You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pegasus.apache.org by wa...@apache.org on 2023/06/21 04:24:29 UTC
[incubator-pegasus] 16/38: feat(new_metrics): add backup-policy-level metric entity and migrate backup-policy-level metrics for meta_backup_service (#1438)
This is an automated email from the ASF dual-hosted git repository.
wangdan pushed a commit to branch migrate-metrics-dev
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
commit 1bf76a4bc832f714492a20655b9eddcc093b6c5e
Author: Dan Wang <wa...@apache.org>
AuthorDate: Sat Apr 15 23:10:21 2023 +0800
feat(new_metrics): add backup-policy-level metric entity and migrate backup-policy-level metrics for meta_backup_service (#1438)
https://github.com/apache/incubator-pegasus/issues/1331
In perf counters, there's only one metric for meta_backup_service, namely
the recent backup duration for each policy, which means this metric is
policy-level. Therefore policy-level entity would also be implemented in
new metrics.
---
src/meta/meta_backup_service.cpp | 48 +++++++++++++++++++++++++++++++---------
src/meta/meta_backup_service.h | 28 ++++++++++++++++++++---
src/meta/test/backup_test.cpp | 3 +--
src/utils/metrics.h | 1 +
4 files changed, 65 insertions(+), 15 deletions(-)
diff --git a/src/meta/meta_backup_service.cpp b/src/meta/meta_backup_service.cpp
index db2afe1d9..cfdddbb53 100644
--- a/src/meta/meta_backup_service.cpp
+++ b/src/meta/meta_backup_service.cpp
@@ -35,7 +35,6 @@
#include "meta/meta_state_service.h"
#include "meta_backup_service.h"
#include "meta_service.h"
-#include "perf_counter/perf_counter.h"
#include "runtime/api_layer1.h"
#include "runtime/rpc/rpc_address.h"
#include "runtime/rpc/rpc_holder.h"
@@ -50,14 +49,47 @@
#include "utils/chrono_literals.h"
#include "utils/flags.h"
#include "utils/fmt_logging.h"
+#include "utils/string_view.h"
#include "utils/time_utils.h"
+METRIC_DEFINE_entity(backup_policy);
+
+METRIC_DEFINE_gauge_int64(backup_policy,
+ policy_recent_backup_duration_ms,
+ dsn::metric_unit::kMilliSeconds,
+ "The recent backup duration");
+
namespace dsn {
namespace replication {
DSN_DECLARE_int32(cold_backup_checkpoint_reserve_minutes);
DSN_DECLARE_int32(fd_lease_seconds);
+namespace {
+
+metric_entity_ptr instantiate_backup_policy_metric_entity(const std::string &policy_name)
+{
+ auto entity_id = fmt::format("backup_policy_{}", policy_name);
+
+ return METRIC_ENTITY_backup_policy.instantiate(entity_id, {{"policy_name", policy_name}});
+}
+
+} // anonymous namespace
+
+backup_policy_metrics::backup_policy_metrics(const std::string &policy_name)
+ : _backup_policy_metric_entity(instantiate_backup_policy_metric_entity(policy_name)),
+ METRIC_VAR_INIT_backup_policy(policy_recent_backup_duration_ms)
+{
+}
+
+const metric_entity_ptr &backup_policy_metrics::backup_policy_metric_entity() const
+{
+ CHECK_NOTNULL(_backup_policy_metric_entity,
+ "backup_policy metric entity should has been instantiated: "
+ "uninitialized entity cannot be used to instantiate metric");
+ return _backup_policy_metric_entity;
+}
+
// TODO: backup_service and policy_context should need two locks, its own _lock and server_state's
// _lock this maybe lead to deadlock, should refactor this
@@ -830,12 +862,8 @@ void policy_context::start()
continue_current_backup_unlocked();
}
- std::string counter_name = _policy.policy_name + ".recent.backup.duration(ms)";
- _counter_policy_recent_backup_duration_ms.init_app_counter(
- "eon.meta.policy",
- counter_name.c_str(),
- COUNTER_TYPE_NUMBER,
- "policy recent backup duration time");
+ CHECK(!_policy.policy_name.empty(), "policy_name should has been initialized");
+ _metrics = std::make_unique<backup_policy_metrics>(_policy.policy_name);
issue_gc_backup_info_task_unlocked();
LOG_INFO("{}: start gc backup info task succeed", _policy.policy_name);
@@ -1011,7 +1039,7 @@ void policy_context::issue_gc_backup_info_task_unlocked()
last_backup_duration_time_ms = (_cur_backup.end_time_ms - _cur_backup.start_time_ms);
}
}
- _counter_policy_recent_backup_duration_ms->set(last_backup_duration_time_ms);
+ METRIC_SET(*_metrics, policy_recent_backup_duration_ms, last_backup_duration_time_ms);
}
void policy_context::sync_remove_backup_info(const backup_info &info, dsn::task_ptr sync_callback)
@@ -1202,7 +1230,7 @@ error_code backup_service::sync_policies_from_remote_storage()
std::shared_ptr<policy_context> policy_ctx = _factory(this);
policy tpolicy;
dsn::json::json_forwarder<policy>::decode(value, tpolicy);
- policy_ctx->set_policy(std::move(tpolicy));
+ policy_ctx->set_policy(tpolicy);
{
zauto_lock l(_lock);
@@ -1334,7 +1362,7 @@ void backup_service::add_backup_policy(dsn::message_ex *msg)
p.start_time.parse_from(request.start_time);
p.app_ids = app_ids;
p.app_names = app_names;
- policy_context_ptr->set_policy(std::move(p));
+ policy_context_ptr->set_policy(p);
do_add_policy(msg, policy_context_ptr, response.hint_message);
}
diff --git a/src/meta/meta_backup_service.h b/src/meta/meta_backup_service.h
index e6563dbcc..021b99552 100644
--- a/src/meta/meta_backup_service.h
+++ b/src/meta/meta_backup_service.h
@@ -36,16 +36,19 @@
#include "common/json_helper.h"
#include "common/replication_other_types.h"
#include "meta_rpc_types.h"
-#include "perf_counter/perf_counter_wrapper.h"
#include "runtime/task/task.h"
#include "runtime/task/task_tracker.h"
#include "utils/api_utilities.h"
+#include "utils/autoref_ptr.h"
#include "utils/error_code.h"
+#include "utils/metrics.h"
+#include "utils/ports.h"
#include "utils/zlocks.h"
namespace dsn {
class message_ex;
class rpc_address;
+
namespace dist {
namespace block_service {
class block_filesystem;
@@ -173,6 +176,23 @@ struct backup_start_time
DEFINE_JSON_SERIALIZATION(hour, minute)
};
+class backup_policy_metrics
+{
+public:
+ backup_policy_metrics() = default;
+ backup_policy_metrics(const std::string &policy_name);
+
+ const metric_entity_ptr &backup_policy_metric_entity() const;
+
+ METRIC_DEFINE_SET(policy_recent_backup_duration_ms, int64_t)
+
+private:
+ const metric_entity_ptr _backup_policy_metric_entity;
+ METRIC_VAR_DECLARE_gauge_int64(policy_recent_backup_duration_ms);
+
+ DISALLOW_COPY_AND_ASSIGN(backup_policy_metrics);
+};
+
//
// the backup process of meta server:
// 1, write the app metadata to block filesystem
@@ -193,6 +213,7 @@ public:
int32_t backup_history_count_to_keep;
bool is_disable;
backup_start_time start_time;
+
policy()
: app_ids(),
backup_interval_seconds(0),
@@ -327,8 +348,9 @@ mock_private :
backup_progress _progress;
std::string _backup_sig; // policy_name@backup_id, used when print backup related log
- perf_counter_wrapper _counter_policy_recent_backup_duration_ms;
-//clang-format on
+ std::unique_ptr<backup_policy_metrics> _metrics;
+
+ //clang-format on
dsn::task_tracker _tracker;
};
diff --git a/src/meta/test/backup_test.cpp b/src/meta/test/backup_test.cpp
index 35b88bb97..be6aaf666 100644
--- a/src/meta/test/backup_test.cpp
+++ b/src/meta/test/backup_test.cpp
@@ -29,7 +29,6 @@
#include <set>
#include <string>
#include <thread>
-#include <type_traits>
#include <utility>
#include <vector>
@@ -272,7 +271,7 @@ protected:
_policy.app_names[4] = "app4";
_policy.app_names[6] = "app6";
_mp._backup_service = _service->_backup_handler.get();
- _mp.set_policy(policy(_policy));
+ _mp.set_policy(_policy);
_service->_storage
->create_node(
diff --git a/src/utils/metrics.h b/src/utils/metrics.h
index 377bb58d7..bdc0ccd50 100644
--- a/src/utils/metrics.h
+++ b/src/utils/metrics.h
@@ -168,6 +168,7 @@ class error_code;
#define METRIC_VAR_INIT_disk(name, ...) METRIC_VAR_INIT(name, disk, ##__VA_ARGS__)
#define METRIC_VAR_INIT_table(name, ...) METRIC_VAR_INIT(name, table, ##__VA_ARGS__)
#define METRIC_VAR_INIT_partition(name, ...) METRIC_VAR_INIT(name, partition, ##__VA_ARGS__)
+#define METRIC_VAR_INIT_backup_policy(name, ...) METRIC_VAR_INIT(name, backup_policy, ##__VA_ARGS__)
// Perform increment-related operations on metrics including gauge and counter.
#define METRIC_VAR_INCREMENT_BY(name, x) \
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pegasus.apache.org
For additional commands, e-mail: commits-help@pegasus.apache.org