You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/01/31 15:39:08 UTC
[doris] 10/20: [improvement](metrics) Metrics add all rowset nums and segment nums (#16208)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit df5ff77587ef471e3a685405d21f88ab16f3a427
Author: Xinyi Zou <zo...@gmail.com>
AuthorDate: Mon Jan 30 09:55:32 2023 +0800
[improvement](metrics) Metrics add all rowset nums and segment nums (#16208)
---
be/src/common/daemon.cpp | 6 ++++++
be/src/olap/tablet.h | 11 +++++++++++
be/src/olap/tablet_manager.cpp | 22 ++++++++++++++++++++++
be/src/olap/tablet_manager.h | 3 +++
be/src/util/doris_metrics.cpp | 6 ++++++
be/src/util/doris_metrics.h | 3 +++
6 files changed, 51 insertions(+)
diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp
index 149b2f1ce4..2f6afb99b7 100644
--- a/be/src/common/daemon.cpp
+++ b/be/src/common/daemon.cpp
@@ -49,6 +49,7 @@
#include "exprs/utility_functions.h"
#include "geo/geo_functions.h"
#include "olap/options.h"
+#include "olap/storage_engine.h"
#include "runtime/bufferpool/buffer_pool.h"
#include "runtime/exec_env.h"
#include "runtime/fragment_mgr.h"
@@ -336,6 +337,11 @@ void Daemon::calculate_metrics_thread() {
DorisMetrics::instance()->system_metrics()->get_network_traffic(
&lst_net_send_bytes, &lst_net_receive_bytes);
}
+
+ DorisMetrics::instance()->all_rowset_nums->set_value(
+ StorageEngine::instance()->tablet_manager()->get_rowset_nums());
+ DorisMetrics::instance()->all_segment_nums->set_value(
+ StorageEngine::instance()->tablet_manager()->get_segment_nums());
}
} while (!_stop_background_threads_latch.wait_for(std::chrono::seconds(15)));
}
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index dcbb0097fc..4b444f13ec 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -97,6 +97,7 @@ public:
size_t num_rows();
int version_count() const;
+ uint64_t segment_count() const;
Version max_version() const;
Version max_version_unlocked() const;
CumulativeCompactionPolicy* cumulative_compaction_policy();
@@ -563,6 +564,7 @@ inline size_t Tablet::num_rows() {
}
inline int Tablet::version_count() const {
+ std::shared_lock rdlock(_meta_lock);
return _tablet_meta->version_count();
}
@@ -570,6 +572,15 @@ inline Version Tablet::max_version() const {
return _tablet_meta->max_version();
}
+inline uint64_t Tablet::segment_count() const {
+ std::shared_lock rdlock(_meta_lock);
+ uint64_t segment_nums = 0;
+ for (auto& rs_meta : _tablet_meta->all_rs_metas()) {
+ segment_nums += rs_meta->num_segments();
+ }
+ return segment_nums;
+}
+
inline Version Tablet::max_version_unlocked() const {
return _tablet_meta->max_version();
}
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 1a5183b080..23c3e7c72c 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -598,6 +598,28 @@ std::vector<TabletSharedPtr> TabletManager::get_all_tablet() {
return res;
}
+uint64_t TabletManager::get_rowset_nums() {
+ uint64_t rowset_nums = 0;
+ for (const auto& tablets_shard : _tablets_shards) {
+ std::shared_lock rdlock(tablets_shard.lock);
+ for (const auto& tablet_map : tablets_shard.tablet_map) {
+ rowset_nums += tablet_map.second->version_count();
+ }
+ }
+ return rowset_nums;
+}
+
+uint64_t TabletManager::get_segment_nums() {
+ uint64_t segment_nums = 0;
+ for (const auto& tablets_shard : _tablets_shards) {
+ std::shared_lock rdlock(tablets_shard.lock);
+ for (const auto& tablet_map : tablets_shard.tablet_map) {
+ segment_nums += tablet_map.second->segment_count();
+ }
+ }
+ return segment_nums;
+}
+
bool TabletManager::get_tablet_id_and_schema_hash_from_path(const string& path,
TTabletId* tablet_id,
TSchemaHash* schema_hash) {
diff --git a/be/src/olap/tablet_manager.h b/be/src/olap/tablet_manager.h
index 75c29bdac7..bbb298cae2 100644
--- a/be/src/olap/tablet_manager.h
+++ b/be/src/olap/tablet_manager.h
@@ -80,6 +80,9 @@ public:
std::vector<TabletSharedPtr> get_all_tablet();
+ uint64_t get_rowset_nums();
+ uint64_t get_segment_nums();
+
// Extract tablet_id and schema_hash from given path.
//
// The normal path pattern is like "/data/{shard_id}/{tablet_id}/{schema_hash}/xxx.data".
diff --git a/be/src/util/doris_metrics.cpp b/be/src/util/doris_metrics.cpp
index 6f61ef7c36..586a66fbb9 100644
--- a/be/src/util/doris_metrics.cpp
+++ b/be/src/util/doris_metrics.cpp
@@ -136,6 +136,9 @@ DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(process_fd_num_limit_hard, MetricUnit::NOUNIT
DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(tablet_cumulative_max_compaction_score, MetricUnit::NOUNIT);
DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(tablet_base_max_compaction_score, MetricUnit::NOUNIT);
+DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(all_rowset_nums, MetricUnit::NOUNIT);
+DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(all_segment_nums, MetricUnit::NOUNIT);
+
DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(compaction_used_permits, MetricUnit::NOUNIT);
DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(compaction_waitting_permits, MetricUnit::NOUNIT);
@@ -259,6 +262,9 @@ DorisMetrics::DorisMetrics() : _metric_registry(_s_registry_name) {
INT_GAUGE_METRIC_REGISTER(_server_metric_entity, tablet_cumulative_max_compaction_score);
INT_GAUGE_METRIC_REGISTER(_server_metric_entity, tablet_base_max_compaction_score);
+ INT_GAUGE_METRIC_REGISTER(_server_metric_entity, all_rowset_nums);
+ INT_GAUGE_METRIC_REGISTER(_server_metric_entity, all_segment_nums);
+
INT_GAUGE_METRIC_REGISTER(_server_metric_entity, compaction_used_permits);
INT_GAUGE_METRIC_REGISTER(_server_metric_entity, compaction_waitting_permits);
diff --git a/be/src/util/doris_metrics.h b/be/src/util/doris_metrics.h
index d117456d01..300c35c9ec 100644
--- a/be/src/util/doris_metrics.h
+++ b/be/src/util/doris_metrics.h
@@ -137,6 +137,9 @@ public:
IntGauge* tablet_cumulative_max_compaction_score;
IntGauge* tablet_base_max_compaction_score;
+ IntGauge* all_rowset_nums;
+ IntGauge* all_segment_nums;
+
// permits have been used for all compaction tasks
IntGauge* compaction_used_permits;
// permits required by the compaction task which is waiting for permits
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org