You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by mg...@apache.org on 2026/09/25 09:02:05 UTC
(kudu) branch master updated: KUDU-3791 add metrics for auto-leader-rebalancer
This is an automated email from the ASF dual-hosted git repository.
martongreber pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new f54a74dbd KUDU-3791 add metrics for auto-leader-rebalancer
f54a74dbd is described below
commit f54a74dbda21190564f0b7dc57dcf6625452debf
Author: Gabriella Lotz <lo...@gmail.com>
AuthorDate: Fri Aug 14 14:21:37 2026 +0200
KUDU-3791 add metrics for auto-leader-rebalancer
Before this change, log grepping was the only way to see what the
auto-leader-rebalancer was doing. Adds six counters on the master:
- auto_leader_rebalancer_moves_scheduled
- auto_leader_rebalancer_moves_completed
- auto_leader_rebalancer_moves_failed
- auto_leader_rebalancer_rounds_completed
- auto_leader_rebalancer_global_pass_skipped
- auto_leader_rebalancer_task_errors (MetricLevel::kWarn)
Naming and units mirror the counters AutoRebalancerTask already
exposes, so the two tasks read as a matched pair on /metrics.
The three move counters keep the invariant
moves_scheduled = moves_completed + moves_failed
over transfers we actually tried, i.e. once we've committed to
sending the LeaderStepDown RPC. Planned moves that turn out to be
stale by the time we get to executing (source tserver deregistered,
destination now in maintenance mode) are skipped without touching
any of the three.
task_errors is warn-severity and meant as the main alerting signal.
It's bumped in RunLoop() when RunLeaderRebalancer() returns an
unexpected error. Losing catalog-manager leadership mid-round is
now swallowed like the first ScopedLeaderSharedLock check already
does, so a leader flap on its own doesn't bump the counter.
Adds a LeaderRebalancerTest.RebalancerMetrics test that exercises
all six counters and asserts the invariant.
Change-Id: I49b86e59043a7f652d4f556b94582bb74c5698ad
Reviewed-on: http://gerrit.cloudera.org:8080/24694
Tested-by: Alexey Serbin <al...@apache.org>
Reviewed-by: Alexey Serbin <al...@apache.org>
Reviewed-by: Marton Greber <gr...@gmail.com>
---
src/kudu/master/auto_leader_rebalancer-test.cc | 103 +++++++++++++++++
src/kudu/master/auto_leader_rebalancer.cc | 148 ++++++++++++++++++++++---
src/kudu/master/auto_leader_rebalancer.h | 21 +++-
src/kudu/master/catalog_manager.cc | 2 +-
4 files changed, 256 insertions(+), 18 deletions(-)
diff --git a/src/kudu/master/auto_leader_rebalancer-test.cc b/src/kudu/master/auto_leader_rebalancer-test.cc
index d350cbab2..8ca3b4c89 100644
--- a/src/kudu/master/auto_leader_rebalancer-test.cc
+++ b/src/kudu/master/auto_leader_rebalancer-test.cc
@@ -52,6 +52,7 @@
#include "kudu/rpc/rpc_controller.h"
#include "kudu/tserver/mini_tablet_server.h"
#include "kudu/tserver/tserver_service.proxy.h" // IWYU pragma: keep
+#include "kudu/util/metrics.h"
#include "kudu/util/monotime.h"
#include "kudu/util/status.h"
#include "kudu/util/test_macros.h"
@@ -87,6 +88,13 @@ DECLARE_uint32(auto_rebalancing_wait_for_replica_moves_seconds);
DECLARE_uint32(leader_rebalancing_max_moves_per_round);
DECLARE_bool(auto_leader_rebalancing_fail_moves_for_test);
+METRIC_DECLARE_counter(auto_leader_rebalancer_moves_scheduled);
+METRIC_DECLARE_counter(auto_leader_rebalancer_moves_completed);
+METRIC_DECLARE_counter(auto_leader_rebalancer_moves_failed);
+METRIC_DECLARE_counter(auto_leader_rebalancer_rounds_completed);
+METRIC_DECLARE_counter(auto_leader_rebalancer_global_pass_skipped);
+METRIC_DECLARE_counter(auto_leader_rebalancer_task_errors);
+
namespace kudu {
namespace master {
@@ -303,6 +311,14 @@ class LeaderRebalancerTest : public KuduTest {
->auto_leader_rebalancer()->moves_scheduled_this_round_for_test_.load();
}
+ // Returns the current value of a server-scoped counter on the leader master.
+ // Instantiate() is idempotent: it returns the existing counter registered by
+ // the AutoLeaderRebalancerTask constructor rather than a fresh zero one.
+ int64_t GetLeaderMasterCounterValue(CounterPrototype* prototype) const {
+ return prototype->Instantiate(
+ cluster_->mini_master()->master()->metric_entity())->value();
+ }
+
// Sums each tserver's leader count across all of 'table_names'.
Status GlobalLeaderDistribution(const vector<string>& table_names,
std::map<string, int32_t>* totals) {
@@ -1157,5 +1173,92 @@ TEST_P(FilterSoftDeletedTableTest, TestFilterSofteDeletedTable) {
}
}
+// Covers all six counters wired in auto_leader_rebalancer.cc:
+// * rounds_completed advances each time RunLeaderRebalancer runs on the leader
+// * moves_scheduled advances on a round that plans transfers
+// * moves_completed advances once moves are allowed to succeed (round 2+)
+// * moves_failed advances when RPCs are forced to fail
+// * global_pass_skipped advances on a round where per-table balancing was busy
+// * task_errors stays flat while RunLeaderRebalancer() returns OK
+TEST_F(LeaderRebalancerTest, RebalancerMetrics) {
+ const int kNumTServers = 3;
+ const int kNumTablets = 6;
+ cluster_opts_.num_tablet_servers = kNumTServers;
+ ASSERT_OK(CreateAndStartCluster());
+ CreateWorkloadTable(kNumTablets, /*num_replicas*/ 3);
+
+ // Pile every leader onto ts0 so the rebalancer has transfers to schedule and
+ // per-table balancing has work this round.
+ const string ts0_uuid = cluster_->mini_tablet_server(0)->uuid();
+ ASSERT_OK(MakeLeaderDistribution({kNumTablets, 0, 0}, table_name()));
+ using LeaderMap = std::map<string, int32_t>;
+ ASSERT_EVENTUALLY([&] {
+ LeaderMap leader_map;
+ ASSERT_OK(GetLeaderDistribution(&leader_map, table_name()));
+ ASSERT_GT(leader_map[ts0_uuid], kNumTablets / kNumTServers);
+ });
+
+ master::AutoLeaderRebalancerTask* leader_rebalancer =
+ cluster_->mini_master()->master()->catalog_manager()->auto_leader_rebalancer();
+
+ // Round 1: force every step-down to fail. Moves are planned (so
+ // per-table balancing is busy), each RPC fails, and the global pass is
+ // skipped because per-table still had work to schedule.
+ const int64_t rounds_before = GetLeaderMasterCounterValue(
+ &METRIC_auto_leader_rebalancer_rounds_completed);
+ const int64_t scheduled_before = GetLeaderMasterCounterValue(
+ &METRIC_auto_leader_rebalancer_moves_scheduled);
+ const int64_t completed_before = GetLeaderMasterCounterValue(
+ &METRIC_auto_leader_rebalancer_moves_completed);
+ const int64_t failed_before = GetLeaderMasterCounterValue(
+ &METRIC_auto_leader_rebalancer_moves_failed);
+ const int64_t skipped_before = GetLeaderMasterCounterValue(
+ &METRIC_auto_leader_rebalancer_global_pass_skipped);
+ const int64_t task_errors_before = GetLeaderMasterCounterValue(
+ &METRIC_auto_leader_rebalancer_task_errors);
+
+ FLAGS_auto_leader_rebalancing_fail_moves_for_test = true;
+ ASSERT_OK(leader_rebalancer->RunLeaderRebalancer());
+
+ const int64_t rounds_after_r1 = GetLeaderMasterCounterValue(
+ &METRIC_auto_leader_rebalancer_rounds_completed);
+ const int64_t scheduled_after_r1 = GetLeaderMasterCounterValue(
+ &METRIC_auto_leader_rebalancer_moves_scheduled);
+ const int64_t completed_after_r1 = GetLeaderMasterCounterValue(
+ &METRIC_auto_leader_rebalancer_moves_completed);
+ const int64_t failed_after_r1 = GetLeaderMasterCounterValue(
+ &METRIC_auto_leader_rebalancer_moves_failed);
+ const int64_t skipped_after_r1 = GetLeaderMasterCounterValue(
+ &METRIC_auto_leader_rebalancer_global_pass_skipped);
+ const int64_t task_errors_after_r1 = GetLeaderMasterCounterValue(
+ &METRIC_auto_leader_rebalancer_task_errors);
+
+ ASSERT_EQ(rounds_before + 1, rounds_after_r1);
+ ASSERT_GT(failed_after_r1, failed_before);
+ // No successful moves this round: every RPC was forced to fail.
+ ASSERT_EQ(completed_before, completed_after_r1);
+ // But the transfers we tried to issue are counted as 'scheduled'.
+ ASSERT_GT(scheduled_after_r1, scheduled_before);
+ // The invariant 'scheduled = completed + failed' holds over this round's
+ // deltas.
+ ASSERT_EQ(scheduled_after_r1 - scheduled_before,
+ (completed_after_r1 - completed_before) +
+ (failed_after_r1 - failed_before));
+ // Per-table balancing scheduled moves, so the global pass was gated off.
+ ASSERT_EQ(skipped_before + 1, skipped_after_r1);
+ // RunLeaderRebalancer() returned OK, so the warning-severity counter for
+ // unexpected failures must not advance.
+ ASSERT_EQ(task_errors_before, task_errors_after_r1);
+
+ // Round 2+: allow moves to succeed. Rerun until moves_completed advances,
+ // proving successful transfers are counted.
+ FLAGS_auto_leader_rebalancing_fail_moves_for_test = false;
+ ASSERT_EVENTUALLY([&] {
+ ASSERT_OK(leader_rebalancer->RunLeaderRebalancer());
+ ASSERT_GT(GetLeaderMasterCounterValue(
+ &METRIC_auto_leader_rebalancer_moves_completed), completed_before);
+ });
+}
+
} // namespace master
} // namespace kudu
diff --git a/src/kudu/master/auto_leader_rebalancer.cc b/src/kudu/master/auto_leader_rebalancer.cc
index 52b51e096..dedcfcd61 100644
--- a/src/kudu/master/auto_leader_rebalancer.cc
+++ b/src/kudu/master/auto_leader_rebalancer.cc
@@ -52,6 +52,7 @@
#include "kudu/tserver/tserver.pb.h"
#include "kudu/util/cow_object.h"
#include "kudu/util/flag_tags.h"
+#include "kudu/util/metrics.h"
#include "kudu/util/monotime.h"
#include "kudu/util/net/net_util.h"
#include "kudu/util/net/sockaddr.h"
@@ -105,15 +106,87 @@ TAG_FLAG(auto_leader_rebalancing_fail_moves_for_test, hidden);
DECLARE_bool(auto_leader_rebalancing_enabled);
+METRIC_DEFINE_counter(server, auto_leader_rebalancer_moves_scheduled,
+ "Auto-Leader-Rebalancer Moves Scheduled",
+ kudu::MetricUnit::kTablets,
+ "Number of leader transfers the auto-leader-rebalancer "
+ "attempted, across the per-table and global passes. "
+ "'Attempted' means the transfer made it past the "
+ "pre-flight applicability checks (target still known "
+ "and not in maintenance mode) and got as far as issuing "
+ "the LeaderStepDown RPC. Together with the "
+ "'_moves_completed' and '_moves_failed' counters the "
+ "invariant 'scheduled = completed + failed' holds.",
+ kudu::MetricLevel::kInfo);
+
+METRIC_DEFINE_counter(server, auto_leader_rebalancer_moves_completed,
+ "Auto-Leader-Rebalancer Moves Completed",
+ kudu::MetricUnit::kTablets,
+ "Number of leader transfers the auto-leader-rebalancer "
+ "issued that were accepted by the target tablet server "
+ "(the LeaderStepDown RPC returned success and the "
+ "response carried no error).",
+ kudu::MetricLevel::kInfo);
+
+METRIC_DEFINE_counter(server, auto_leader_rebalancer_moves_failed,
+ "Auto-Leader-Rebalancer Moves Failed",
+ kudu::MetricUnit::kTablets,
+ "Number of leader transfers attempted by the "
+ "auto-leader-rebalancer that failed at the RPC layer or "
+ "returned an error from the target tablet server.",
+ kudu::MetricLevel::kInfo);
+
+METRIC_DEFINE_counter(server, auto_leader_rebalancer_rounds_completed,
+ "Auto-Leader-Rebalancer Rounds Completed",
+ kudu::MetricUnit::kUnits,
+ "Number of full rebalancing cycles completed by the "
+ "auto-leader-rebalancer (only counts rounds where this "
+ "master was the catalog manager leader).",
+ kudu::MetricLevel::kInfo);
+
+METRIC_DEFINE_counter(server, auto_leader_rebalancer_global_pass_skipped,
+ "Auto-Leader-Rebalancer Global Pass Skipped",
+ kudu::MetricUnit::kUnits,
+ "Number of rounds where the global corrective pass was "
+ "skipped because the per-table pass still had work to do. "
+ "If this counter grows without ever pausing, per-table "
+ "balancing is starving the global pass.",
+ kudu::MetricLevel::kInfo);
+
+METRIC_DEFINE_counter(server, auto_leader_rebalancer_task_errors,
+ "Auto-Leader-Rebalancer Task Errors",
+ kudu::MetricUnit::kUnits,
+ "Number of auto-leader-rebalancer rounds that returned "
+ "an unexpected error (any non-OK status from "
+ "RunLeaderRebalancer() other than this master losing "
+ "catalog-manager leadership mid-round). Suitable as the "
+ "primary alerting signal for 'the auto-leader-rebalancer "
+ "task is faltering'.",
+ kudu::MetricLevel::kWarn);
+
namespace kudu {
namespace master {
-AutoLeaderRebalancerTask::AutoLeaderRebalancerTask(CatalogManager* catalog_manager,
- TSManager* ts_manager)
+AutoLeaderRebalancerTask::AutoLeaderRebalancerTask(
+ CatalogManager* catalog_manager,
+ TSManager* ts_manager,
+ const scoped_refptr<MetricEntity>& metric_entity)
: catalog_manager_(catalog_manager),
ts_manager_(ts_manager),
shutdown_(1),
random_generator_(random_device_()),
+ moves_scheduled_(
+ METRIC_auto_leader_rebalancer_moves_scheduled.Instantiate(metric_entity)),
+ moves_completed_(
+ METRIC_auto_leader_rebalancer_moves_completed.Instantiate(metric_entity)),
+ moves_failed_(
+ METRIC_auto_leader_rebalancer_moves_failed.Instantiate(metric_entity)),
+ rounds_completed_(
+ METRIC_auto_leader_rebalancer_rounds_completed.Instantiate(metric_entity)),
+ global_pass_skipped_(
+ METRIC_auto_leader_rebalancer_global_pass_skipped.Instantiate(metric_entity)),
+ task_errors_(
+ METRIC_auto_leader_rebalancer_task_errors.Instantiate(metric_entity)),
number_of_loop_iterations_for_test_(0),
moves_scheduled_this_round_for_test_(0) {}
@@ -407,9 +480,12 @@ Status AutoLeaderRebalancerTask::RunLeaderRebalanceForTable(
moves_scheduled_this_round_for_test_.load(),
leader_transfer_source.size());
// Step 4. Do Leader transfer tasks.
- // @TODO(duyuqi), optimal speed
- // If leader rebalancing tasks is too many, each rpc of the thread wait the response
- // synchronously, which may be very slow.
+ //
+ // TODO(gabriellalotz): each LeaderStepDown RPC below is issued synchronously,
+ // so a round with many transfers can be slow. If we change this loop to
+ // dispatch RPCs asynchronously in a follow-up, the moves_scheduled /
+ // moves_completed / moves_failed counters let us verify the async version
+ // isn't dropping moves.
int leader_transfer_count = 0;
for (const auto& task : leader_transfer_tasks) {
@@ -424,6 +500,12 @@ Status AutoLeaderRebalancerTask::RunLeaderRebalanceForTable(
RpcController rpc;
rpc.set_timeout(MonoDelta::FromSeconds(FLAGS_auto_leader_rebalancing_rpc_timeout_seconds));
+ // Pre-flight checks: if the world has changed since we planned these
+ // moves (the source tserver deregistered, or the destination is now in
+ // maintenance mode), the transfer isn't applicable anymore. Skip it and
+ // let the next round replan. These skips don't touch moves_scheduled or
+ // moves_failed, since they aren't actual attempts. Keeps the invariant
+ // 'scheduled = completed + failed' scoped to moves we actually tried.
auto* host_port = FindOrNull(host_port_by_leader_ts_uuid, leader_uuid);
if (!host_port) {
continue;
@@ -437,10 +519,12 @@ Status AutoLeaderRebalancerTask::RunLeaderRebalanceForTable(
continue;
}
+ moves_scheduled_->Increment();
vector<Sockaddr> resolved;
if (Status s = host_port->ResolveAddresses(&resolved); !s.ok()) {
WARN_NOT_OK(s, Substitute("leader transfer for tablet $0: could not resolve $1",
task.first, host_port->ToString()));
+ moves_failed_->Increment();
continue;
}
ConsensusServiceProxy proxy(messenger_, resolved[0], host_port->host());
@@ -454,23 +538,24 @@ Status AutoLeaderRebalancerTask::RunLeaderRebalanceForTable(
if (!s.ok()) {
WARN_NOT_OK(s, Substitute("leader transfer for tablet $0 from $1 to $2 failed",
task.first, leader_uuid, task.second.second));
+ moves_failed_->Increment();
continue;
}
if (!response.has_error()) {
leader_transfer_count++;
+ moves_completed_->Increment();
VLOG(1) << Substitute("leader transfer table: $0, tablet_id: $1, from: $2 to: $3",
table_data.name(),
task.first,
leader_uuid,
task.second.second);
} else {
+ moves_failed_->Increment();
LOG(WARNING) << Substitute(
"leader transfer for tablet $0 (from $1 to $2) failed: $3",
task.first, leader_uuid, task.second.second, response.error().ShortDebugString());
}
}
- // @TODO(duyuqi)
- // Add metrics to replace the log.
VLOG(0) << Substitute("table: $0, leader rebalance finish, leader transfer count: $1",
table_data.name(),
leader_transfer_count);
@@ -693,6 +778,9 @@ Status AutoLeaderRebalancerTask::RunGlobalLeaderRebalance(
const string& tablet_id = task.first;
const string& leader_uuid = task.second.first;
const string& dest_uuid = task.second.second;
+ // Pre-flight applicability checks: see the matching block in
+ // RunLeaderRebalanceForTable() for why these skips don't touch either
+ // the '_moves_scheduled' or '_moves_failed' counter.
if (PREDICT_FALSE(TServerStatePB::MAINTENANCE_MODE ==
ts_manager_->GetTServerState(dest_uuid))) {
continue;
@@ -716,6 +804,7 @@ Status AutoLeaderRebalancerTask::RunGlobalLeaderRebalance(
RpcController rpc;
rpc.set_timeout(MonoDelta::FromSeconds(FLAGS_auto_leader_rebalancing_rpc_timeout_seconds));
+ moves_scheduled_->Increment();
// This pass is best-effort: a transfer may fail transiently (for example
// "already in progress" if a previous round's move for this tablet has
// not settled yet). Log and move on rather than aborting the round, since
@@ -724,6 +813,7 @@ Status AutoLeaderRebalancerTask::RunGlobalLeaderRebalance(
Status s = host_port->ResolveAddresses(&resolved);
if (!s.ok()) {
WARN_NOT_OK(s, Substitute("global leader rebalance: cannot resolve $0", leader_uuid));
+ moves_failed_->Increment();
continue;
}
ConsensusServiceProxy proxy(messenger_, resolved[0], host_port->host());
@@ -731,13 +821,16 @@ Status AutoLeaderRebalancerTask::RunGlobalLeaderRebalance(
if (!s.ok()) {
WARN_NOT_OK(s, Substitute(
"global leader rebalance: leader step down for tablet $0 failed", tablet_id));
+ moves_failed_->Increment();
continue;
}
if (!response.has_error()) {
+ moves_completed_->Increment();
VLOG(1) << Substitute(
"global leader rebalance transfer table: $0, tablet_id: $1, from: $2 to: $3",
table_data.name(), tablet_id, leader_uuid, dest_uuid);
} else {
+ moves_failed_->Increment();
LOG(WARNING) << Substitute(
"global leader rebalance: transfer for tablet $0 (from $1 to $2) failed: $3",
tablet_id, leader_uuid, dest_uuid, response.error().ShortDebugString());
@@ -797,7 +890,16 @@ Status AutoLeaderRebalancerTask::RunLeaderRebalancer() {
vector<scoped_refptr<TableInfo>> table_infos;
{
CatalogManager::ScopedLeaderSharedLock leader_lock(catalog_manager_);
- RETURN_NOT_OK(leader_lock.first_failed_status());
+ // Mirror the earlier check: losing catalog-manager leadership between the
+ // top-of-round check and here is expected, not an error. Report OK so
+ // callers can distinguish that from real failures (which do increment
+ // 'auto_leader_rebalancer_task_errors').
+ if (!leader_lock.first_failed_status().ok()) {
+ VLOG(1) << "auto-leader-rebalancer: lost catalog-manager leadership "
+ "mid-round, skipping remainder of this round: "
+ << leader_lock.first_failed_status().ToString();
+ return Status::OK();
+ }
if (FLAGS_leader_rebalancing_ignore_soft_deleted_tables) {
catalog_manager_->GetNormalizedTables(&table_infos);
} else {
@@ -831,16 +933,23 @@ Status AutoLeaderRebalancerTask::RunLeaderRebalancer() {
// moved (the leadership transfers are asynchronous), and a follow-up round
// does the global correction with a consistent view instead.
//
- // TODO(gabriellalotz): if per-table balancing keeps finding work every round
- // (e.g. from continuous leader elections), the global pass can be starved.
- // Emit a metric when it's skipped for this reason so the situation is
- // diagnosable.
+ // If per-table balancing keeps finding work every round (e.g. from continuous
+ // leader elections), the global pass can be starved: the
+ // 'auto_leader_rebalancer_global_pass_skipped' counter, incremented below,
+ // exposes that so operators can spot the situation.
+ //
+ // Note the gate below counts planned moves, while the
+ // 'auto_leader_rebalancer_moves_scheduled' metric counts only the transfers
+ // that clear the pre-flight checks. In a round where every planned move is
+ // skipped in pre-flight, global_pass_skipped ticks up while moves_scheduled
+ // stays flat: the two are scoped differently, not a lost move.
if (per_table_moves_scheduled == 0) {
RETURN_NOT_OK(RunGlobalLeaderRebalance(
table_infos, tserver_uuids, exclude_dest_uuids, &global_leader_count_by_ts_uuid));
+ } else {
+ global_pass_skipped_->Increment();
}
- // @TODO(duyuqi)
- // Enrich the log and add metrics for leader rebalancer.
+ rounds_completed_->Increment();
LOG(INFO) << "All tables' leader rebalancing finished this round";
return Status::OK();
}
@@ -849,8 +958,15 @@ void AutoLeaderRebalancerTask::RunLoop() {
while (
!shutdown_.WaitFor(MonoDelta::FromSeconds(FLAGS_auto_leader_rebalancing_interval_seconds))) {
if (FLAGS_auto_leader_rebalancing_enabled) {
- WARN_NOT_OK(RunLeaderRebalancer(),
- Substitute("the master instance isn't leader"));
+ // RunLeaderRebalancer() normalises the "we're not the catalog-manager
+ // leader" cases to Status::OK(), so any non-OK reaching here is an
+ // unexpected failure worth surfacing to both the log and the
+ // warning-severity 'auto_leader_rebalancer_task_errors' counter.
+ const Status s = RunLeaderRebalancer();
+ if (PREDICT_FALSE(!s.ok())) {
+ task_errors_->Increment();
+ WARN_NOT_OK(s, "auto-leader-rebalancer round failed");
+ }
}
}
}
diff --git a/src/kudu/master/auto_leader_rebalancer.h b/src/kudu/master/auto_leader_rebalancer.h
index 6577044e6..850a9c9bf 100644
--- a/src/kudu/master/auto_leader_rebalancer.h
+++ b/src/kudu/master/auto_leader_rebalancer.h
@@ -32,6 +32,8 @@
namespace kudu {
+class Counter;
+class MetricEntity;
class Thread;
namespace rpc {
@@ -54,7 +56,9 @@ class TableInfo;
// configured poll period. It performs no work when the master is a follower.
class AutoLeaderRebalancerTask {
public:
- AutoLeaderRebalancerTask(CatalogManager* catalog_manager, TSManager* ts_manager);
+ AutoLeaderRebalancerTask(CatalogManager* catalog_manager,
+ TSManager* ts_manager,
+ const scoped_refptr<MetricEntity>& metric_entity);
~AutoLeaderRebalancerTask();
@@ -131,6 +135,21 @@ class AutoLeaderRebalancerTask {
std::random_device random_device_;
std::mt19937 random_generator_;
+ // Counters exposing what the auto-leader-rebalancer is doing across rounds.
+ // See the METRIC_DEFINE_counter blocks in auto_leader_rebalancer.cc for the
+ // per-metric descriptions. The three move counters follow the invariant
+ // moves_scheduled_ = moves_completed_ + moves_failed_
+ // over the set of transfers that made it past the pre-flight applicability
+ // checks in the per-table and global passes.
+ scoped_refptr<Counter> moves_scheduled_;
+ scoped_refptr<Counter> moves_completed_;
+ scoped_refptr<Counter> moves_failed_;
+ scoped_refptr<Counter> rounds_completed_;
+ scoped_refptr<Counter> global_pass_skipped_;
+ // Warn-severity counter, meant as the main signal for alerting. Bumped
+ // once per round that returned an unexpected error.
+ scoped_refptr<Counter> task_errors_;
+
// Variables for testing.
std::atomic<int> number_of_loop_iterations_for_test_;
std::atomic<int> moves_scheduled_this_round_for_test_;
diff --git a/src/kudu/master/catalog_manager.cc b/src/kudu/master/catalog_manager.cc
index c400d59cb..49c4d1da8 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -1077,7 +1077,7 @@ Status CatalogManager::Init(bool is_first_run) {
// every table.
unique_ptr<AutoLeaderRebalancerTask> leader_task(
- new AutoLeaderRebalancerTask(this, master_->ts_manager()));
+ new AutoLeaderRebalancerTask(this, master_->ts_manager(), master_->metric_entity()));
RETURN_NOT_OK_PREPEND(leader_task->Init(),
"failed thie initialize auto-leader-rebalancing task");
auto_leader_rebalancer_ = std::move(leader_task);