You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by al...@apache.org on 2026/08/13 23:00:26 UTC
(kudu) branch master updated: [tools/master] make move of a leader replica recoverable
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new c667e360f [tools/master] make move of a leader replica recoverable
c667e360f is described below
commit c667e360fcbe6a4a10338cf690e4792b7252d449
Author: zhangyifan27 <ch...@163.com>
AuthorDate: Fri Jul 24 18:59:33 2026 +0800
[tools/master] make move of a leader replica recoverable
Moving a replica ('kudu tablet change_config move_replica' or 'kudu cluster
rebalance') is client-driven: the source is marked 'replace' and, if it is
the leader, the client asks it to step down so it can be evicted (Raft can't
evict a leader). If the client is interrupted (Ctrl+C, or an aborted/timed-out
rebalance) after the config change but before the leader steps down, the
tablet is left over-replicated with a stuck, un-evictable leader.
Add two recovery paths:
* Master-side self-healing: once a leader marked 'replace' has its replacement
promoted to voter, ProcessTabletReport() schedules an
AsyncLeaderStepDownForReplacementTask to step it down so it can be evicted
(see ShouldStepDownLeaderForReplacement(); guarded by
--catalog_manager_step_down_leader_for_replacement, on by default, 3-4-3
only). Driven by tablet state, so it recovers an interrupted move_replica or
'cluster rebalance' with no client involved.
* CLI idempotency (move_replica): IsReplicaMoveScheduled() lets MoveReplica()
detect an already-scheduled move and wait for it, so re-running resumes
instead of failing on the over-replicated tablet.
Change-Id: Ic7b28ca4e41355bd82c76d50e137f06197b402bc
Co-authored-by: Cursor <cu...@cursor.com>
Reviewed-on: http://gerrit.cloudera.org:8080/24630
Tested-by: Alexey Serbin <al...@apache.org>
Reviewed-by: Alexey Serbin <al...@apache.org>
---
src/kudu/consensus/quorum_util-test.cc | 63 ++++++++++
src/kudu/consensus/quorum_util.cc | 35 ++++++
src/kudu/consensus/quorum_util.h | 14 +++
.../raft_consensus_nonvoter-itest.cc | 109 ++++++++++++++++++
src/kudu/master/catalog_manager.cc | 127 +++++++++++++++++++++
src/kudu/tools/kudu-admin-test.cc | 92 +++++++++++++++
src/kudu/tools/tool_action_tablet.cc | 32 ++++--
src/kudu/tools/tool_replica_util.cc | 46 ++++++++
src/kudu/tools/tool_replica_util.h | 13 +++
9 files changed, 522 insertions(+), 9 deletions(-)
diff --git a/src/kudu/consensus/quorum_util-test.cc b/src/kudu/consensus/quorum_util-test.cc
index 4422feea3..d12803869 100644
--- a/src/kudu/consensus/quorum_util-test.cc
+++ b/src/kudu/consensus/quorum_util-test.cc
@@ -1909,5 +1909,68 @@ TEST(QuorumUtilTest, ReplaceAllTabletReplicas) {
EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));
}
+// Verify ShouldStepDownLeaderForReplacement(): step down a leader marked with
+// 'replace' only once its replacement has been promoted to voter.
+TEST(QuorumUtilTest, ShouldStepDownLeaderForReplacement) {
+ constexpr auto kReplicationFactor = 3;
+
+ {
+ // No leader: nobody to step down.
+ RaftConfigPB config;
+ AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
+ AddPeer(&config, "B", V, '+');
+ AddPeer(&config, "C", V, '+');
+ EXPECT_FALSE(ShouldStepDownLeaderForReplacement(config, "", kReplicationFactor));
+ }
+ {
+ // Leader marked, but no replacement added yet (only RF voters): don't step
+ // down early.
+ RaftConfigPB config;
+ AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
+ AddPeer(&config, "B", V, '+');
+ AddPeer(&config, "C", V, '+');
+ EXPECT_FALSE(ShouldStepDownLeaderForReplacement(config, "A", kReplicationFactor));
+ }
+ {
+ // Replacement present but not yet promoted (still a NON_VOTER): don't step
+ // down.
+ RaftConfigPB config;
+ AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
+ AddPeer(&config, "B", V, '+');
+ AddPeer(&config, "C", V, '+');
+ AddPeer(&config, "D", N, '+', {{"PROMOTE", true}});
+ EXPECT_FALSE(ShouldStepDownLeaderForReplacement(config, "A", kReplicationFactor));
+ }
+ {
+ // Replacement promoted to voter (RF+1 healthy voters): step down.
+ RaftConfigPB config;
+ AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
+ AddPeer(&config, "B", V, '+');
+ AddPeer(&config, "C", V, '+');
+ AddPeer(&config, "D", V, '+');
+ EXPECT_TRUE(ShouldStepDownLeaderForReplacement(config, "A", kReplicationFactor));
+ }
+ {
+ // A non-leader is marked (RF+1 voters): don't step down; it is evicted
+ // directly instead.
+ RaftConfigPB config;
+ AddPeer(&config, "A", V, '+');
+ AddPeer(&config, "B", V, '+', {{"REPLACE", true}});
+ AddPeer(&config, "C", V, '+');
+ AddPeer(&config, "D", V, '+');
+ EXPECT_FALSE(ShouldStepDownLeaderForReplacement(config, "A", kReplicationFactor));
+ EXPECT_TRUE(ShouldEvictReplica(config, "A", kReplicationFactor));
+ }
+ {
+ // Fewer than RF healthy voters: don't step down (eviction couldn't commit).
+ RaftConfigPB config;
+ AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
+ AddPeer(&config, "B", V, '+');
+ AddPeer(&config, "C", V, '-');
+ AddPeer(&config, "D", V, '-');
+ EXPECT_FALSE(ShouldStepDownLeaderForReplacement(config, "A", kReplicationFactor));
+ }
+}
+
} // namespace consensus
} // namespace kudu
diff --git a/src/kudu/consensus/quorum_util.cc b/src/kudu/consensus/quorum_util.cc
index 7b7524b03..555c91fa6 100644
--- a/src/kudu/consensus/quorum_util.cc
+++ b/src/kudu/consensus/quorum_util.cc
@@ -837,5 +837,40 @@ bool ShouldEvictReplica(const RaftConfigPB& config,
return should_evict;
}
+bool ShouldStepDownLeaderForReplacement(const RaftConfigPB& config,
+ const string& leader_uuid,
+ int replication_factor) {
+ if (leader_uuid.empty()) {
+ return false;
+ }
+
+ bool leader_with_replace = false;
+ int num_voters_total = 0;
+ int num_voters_healthy = 0;
+ for (const RaftPeerPB& peer : config.peers()) {
+ if (peer.member_type() != RaftPeerPB::VOTER) {
+ continue;
+ }
+ ++num_voters_total;
+ if (peer.health_report().overall_health() == HealthReportPB::HEALTHY) {
+ ++num_voters_healthy;
+ }
+ if (peer.permanent_uuid() == leader_uuid && peer.attrs().replace()) {
+ leader_with_replace = true;
+ }
+ }
+
+ // Step down only if the leader is marked with 'replace' and there are more
+ // than 'replication_factor' healthy voters. The latter ensures the
+ // replacement was already promoted to voter (so we don't step down early) and
+ // that a healthy majority remains to commit the ensuing eviction.
+ const bool should_step_down = leader_with_replace &&
+ num_voters_healthy > replication_factor &&
+ num_voters_healthy >= MajoritySize(num_voters_total);
+ VLOG(2) << "decision: should" << (should_step_down ? "" : " not")
+ << " step down leader " << leader_uuid << " for replacement";
+ return should_step_down;
+}
+
} // namespace consensus
} // namespace kudu
diff --git a/src/kudu/consensus/quorum_util.h b/src/kudu/consensus/quorum_util.h
index b78c9104a..12edd3547 100644
--- a/src/kudu/consensus/quorum_util.h
+++ b/src/kudu/consensus/quorum_util.h
@@ -124,6 +124,20 @@ bool ShouldEvictReplica(const RaftConfigPB& config,
int replication_factor,
std::string* uuid_to_evict = nullptr);
+// Whether the leader replica should be made to step down so that it can be
+// evicted afterwards. Kudu's Raft implementation cannot evict the leader
+// replica (see ShouldEvictReplica()), so a leader marked with the 'replace'
+// attribute must first step down. This lets the master recover when the
+// client-driven step-down (CLI or auto-rebalancer) was interrupted.
+//
+// Returns 'true' when the leader is marked with 'replace' and there are more
+// than 'replication_factor' healthy voters. The latter both keeps a healthy
+// majority for the ensuing eviction and ensures the replacement replica has
+// already been promoted to voter before the leader steps down.
+bool ShouldStepDownLeaderForReplacement(const RaftConfigPB& config,
+ const std::string& leader_uuid,
+ int replication_factor);
+
} // namespace consensus
} // namespace kudu
diff --git a/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc b/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc
index d81aa7bac..e29d514eb 100644
--- a/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc
+++ b/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc
@@ -86,6 +86,7 @@ using kudu::consensus::IsRaftConfigMember;
using kudu::consensus::IsRaftConfigVoter;
using kudu::consensus::RaftPeerPB;
using kudu::itest::AddServer;
+using kudu::itest::BulkChangeConfig;
using kudu::itest::GetConsensusState;
using kudu::itest::GetInt64Metric;
using kudu::itest::GetTableLocations;
@@ -98,6 +99,7 @@ using kudu::itest::TabletServerMap;
using kudu::itest::WAIT_FOR_LEADER;
using kudu::itest::WaitForNumTabletsOnTS;
using kudu::itest::WaitForReplicasReportedToMaster;
+using kudu::itest::WaitUntilCommittedConfigNumVotersIs;
using kudu::master::ANY_REPLICA;
using kudu::master::GetTableLocationsResponsePB;
using kudu::master::GetTabletLocationsResponsePB;
@@ -1246,6 +1248,113 @@ TEST_F(RaftConsensusNonVoterITest, CatalogManagerEvictsExcessNonVoter) {
NO_FATALS(cluster_->AssertNoCrashes());
}
+// With a leader marked 'replace' and no client involved (e.g. an interrupted
+// move_replica/rebalance), the master must add+promote a replacement, step the
+// leader down and evict it. With the mechanism disabled the tablet instead
+// stays stuck over-replicated. See ShouldStepDownLeaderForReplacement().
+class LeaderReplacementRecoveryITest :
+ public RaftConsensusNonVoterITest,
+ public ::testing::WithParamInterface<bool> {
+};
+
+INSTANTIATE_TEST_SUITE_P(, LeaderReplacementRecoveryITest, ::testing::Bool());
+
+TEST_P(LeaderReplacementRecoveryITest, StepDownAndEvictMarkedLeader) {
+ const bool enable_step_down = GetParam();
+ const MonoDelta kTimeout = MonoDelta::FromSeconds(60);
+ const int kReplicasNum = 3;
+ FLAGS_num_replicas = kReplicasNum;
+ // One extra tablet server so the master can add a replacement replica.
+ FLAGS_num_tablet_servers = kReplicasNum + 1;
+
+ const vector<string> kTserverFlags = {
+ "--raft_prepare_replacement_before_eviction=true",
+ // Heartbeat to the master often so it reacts within a few heartbeats; this
+ // makes the negative (disabled) case below trustworthy: if the master were
+ // going to evict the marked leader, it would have many chances to do so
+ // within the bounded wait.
+ "--heartbeat_interval_ms=100",
+ };
+ const vector<string> kMasterFlags = {
+ "--raft_prepare_replacement_before_eviction=true",
+ Substitute("--catalog_manager_step_down_leader_for_replacement=$0",
+ enable_step_down),
+ };
+ NO_FATALS(BuildAndStart(kTserverFlags, kMasterFlags));
+
+ // Committed op in the leader's term is required before changing the config.
+ TServerDetails* leader = nullptr;
+ ASSERT_OK(WaitForLeaderWithCommittedOp(tablet_id_, kTimeout, &leader));
+ const string leader_uuid = leader->uuid();
+
+ // A follower that stays in the config, used to observe the outcome.
+ TServerDetails* observer = nullptr;
+ for (const auto& e : tablet_replicas_) {
+ if (e.first == tablet_id_ && e.second->uuid() != leader_uuid) {
+ observer = e.second;
+ break;
+ }
+ }
+ ASSERT_NE(nullptr, observer);
+
+ // Mark the leader 'replace' (what an interrupted move leaves behind) without
+ // adding a replacement: the master must drive everything from here.
+ consensus::ConsensusStatePB cstate;
+ ASSERT_OK(GetConsensusState(leader, tablet_id_, kTimeout,
+ EXCLUDE_HEALTH_REPORT, &cstate));
+ {
+ vector<consensus::BulkChangeConfigRequestPB::ConfigChangeItemPB> changes;
+ consensus::BulkChangeConfigRequestPB::ConfigChangeItemPB change;
+ change.set_type(consensus::MODIFY_PEER);
+ RaftPeerPB* peer = change.mutable_peer();
+ peer->set_permanent_uuid(leader_uuid);
+ peer->mutable_attrs()->set_replace(true);
+ changes.emplace_back(std::move(change));
+ ASSERT_OK(BulkChangeConfig(leader, tablet_id_, changes, kTimeout,
+ cstate.committed_config().opid_index()));
+ }
+
+ if (enable_step_down) {
+ // Terminal state: the old leader is evicted and the config is back at RF.
+ ASSERT_EVENTUALLY([&] {
+ consensus::ConsensusStatePB cs;
+ ASSERT_OK(GetConsensusState(observer, tablet_id_, kTimeout,
+ EXCLUDE_HEALTH_REPORT, &cs));
+ ASSERT_EQ(kReplicasNum, cs.committed_config().peers_size());
+ for (const auto& peer : cs.committed_config().peers()) {
+ ASSERT_NE(leader_uuid, peer.permanent_uuid())
+ << "the 'replace'-marked leader was not evicted";
+ }
+ });
+ } else {
+ // The replacement is still added and promoted (RF+1 voters), but the marked
+ // leader can never be evicted, so the tablet stays stuck over-replicated.
+ ASSERT_OK(WaitUntilCommittedConfigNumVotersIs(kReplicasNum + 1, observer,
+ tablet_id_, kTimeout));
+ if (AllowSlowTests()) {
+ // With frequent heartbeats the master gets dozens of chances to act over
+ // this window; confirm it never evicts the marked leader, which stays
+ // present and keeps leading.
+ SleepFor(MonoDelta::FromSeconds(5));
+ consensus::ConsensusStatePB cs;
+ ASSERT_OK(GetConsensusState(observer, tablet_id_, kTimeout,
+ EXCLUDE_HEALTH_REPORT, &cs));
+ ASSERT_EQ(kReplicasNum + 1, cs.committed_config().peers_size());
+ ASSERT_EQ(leader_uuid, cs.leader_uuid());
+ bool leader_still_present = false;
+ for (const auto& peer : cs.committed_config().peers()) {
+ if (peer.permanent_uuid() == leader_uuid) {
+ leader_still_present = true;
+ break;
+ }
+ }
+ ASSERT_TRUE(leader_still_present);
+ }
+ }
+
+ NO_FATALS(cluster_->AssertNoCrashes());
+}
+
// Check that the catalog manager adds a non-voter replica to replace failed
// voter replica in a tablet.
//
diff --git a/src/kudu/master/catalog_manager.cc b/src/kudu/master/catalog_manager.cc
index d29383626..c400d59cb 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -302,6 +302,15 @@ DEFINE_bool(catalog_manager_evict_excess_replicas, true,
TAG_FLAG(catalog_manager_evict_excess_replicas, hidden);
TAG_FLAG(catalog_manager_evict_excess_replicas, runtime);
+DEFINE_bool(catalog_manager_step_down_leader_for_replacement, true,
+ "Whether the catalog manager makes a leader replica marked with the "
+ "'replace' attribute step down so it can be evicted. This lets the "
+ "master complete a move of a leader replica even if the client that "
+ "initiated it was interrupted. Only effective with the 3-4-3 replica "
+ "management scheme (--raft_prepare_replacement_before_eviction).");
+TAG_FLAG(catalog_manager_step_down_leader_for_replacement, advanced);
+TAG_FLAG(catalog_manager_step_down_leader_for_replacement, runtime);
+
DEFINE_int32(catalog_manager_inject_latency_list_authz_ms, 0,
"Injects a sleep in milliseconds while authorizing a ListTables "
"request. This is a test-only flag.");
@@ -474,9 +483,13 @@ using kudu::cfile::TypeEncodingInfo;
using kudu::consensus::ConsensusServiceProxy;
using kudu::consensus::ConsensusStatePB;
using kudu::consensus::IsRaftConfigMember;
+using kudu::consensus::LeaderStepDownMode;
+using kudu::consensus::LeaderStepDownRequestPB;
+using kudu::consensus::LeaderStepDownResponsePB;
using kudu::consensus::RaftConfigPB;
using kudu::consensus::RaftConsensus;
using kudu::consensus::RaftPeerPB;
+using kudu::consensus::ShouldStepDownLeaderForReplacement;
using kudu::consensus::StartTabletCopyRequestPB;
using kudu::consensus::kMinimumTerm;
using kudu::hms::HmsClientVerifyKuduSyncConfig;
@@ -5474,6 +5487,112 @@ bool AsyncEvictReplicaTask::SendRequest(int attempt) {
return true;
}
+// Task to make a leader replica marked with the 'replace' attribute step down
+// so that the catalog manager can evict it (Raft cannot evict a leader). See
+// ShouldStepDownLeaderForReplacement().
+//
+// The task targets the specific replica captured at creation time; if that
+// replica is no longer the leader when the RPC is sent, the task completes
+// without doing anything, so it never steps down an unrelated, healthy leader.
+class AsyncLeaderStepDownForReplacementTask : public RetryingTSRpcTask {
+ public:
+ AsyncLeaderStepDownForReplacementTask(Master* master,
+ scoped_refptr<TabletInfo> tablet,
+ string leader_uuid);
+
+ string type_name() const override;
+ string description() const override;
+
+ protected:
+ bool SendRequest(int attempt) override;
+ void HandleResponse(int attempt) override;
+
+ private:
+ const string& tablet_id() const override { return tablet_->id(); }
+
+ const scoped_refptr<TabletInfo> tablet_;
+ const string leader_uuid_;
+
+ LeaderStepDownResponsePB resp_;
+};
+
+AsyncLeaderStepDownForReplacementTask::AsyncLeaderStepDownForReplacementTask(
+ Master* master,
+ scoped_refptr<TabletInfo> tablet,
+ string leader_uuid)
+ : RetryingTSRpcTask(master,
+ unique_ptr<TSPicker>(new PickSpecificUUID(leader_uuid)),
+ tablet->table().get()),
+ tablet_(std::move(tablet)),
+ leader_uuid_(std::move(leader_uuid)) {
+}
+
+string AsyncLeaderStepDownForReplacementTask::type_name() const {
+ return "LeaderStepDownForReplacement";
+}
+
+string AsyncLeaderStepDownForReplacementTask::description() const {
+ return Substitute("$0 RPC for tablet $1 (leader $2)",
+ type_name(), tablet_->id(), leader_uuid_);
+}
+
+bool AsyncLeaderStepDownForReplacementTask::SendRequest(int attempt) {
+ // If the targeted replica is no longer the leader, bail out to avoid asking a
+ // different, possibly healthy, leader to step down.
+ {
+ TabletMetadataLock l(tablet_.get(), LockMode::READ);
+ const auto& current_leader =
+ l.data().pb.consensus_state().leader_uuid();
+ if (current_leader != leader_uuid_) {
+ LOG_WITH_PREFIX(INFO) << Substitute(
+ "replica $0 is no longer the leader (current leader: '$1'); "
+ "no leader step-down needed", leader_uuid_, current_leader);
+ MarkComplete();
+ return false;
+ }
+ }
+
+ LeaderStepDownRequestPB req;
+ req.set_dest_uuid(leader_uuid_);
+ req.set_tablet_id(tablet_->id());
+ req.set_mode(LeaderStepDownMode::GRACEFUL);
+ VLOG(1) << Substitute("Sending $0 request to $1 (attempt $2): $3",
+ type_name(), target_ts_desc_->ToString(), attempt,
+ SecureDebugString(req));
+ consensus_proxy_->LeaderStepDownAsync(req, &resp_, &rpc_,
+ [this]() { this->RpcCallback(); });
+ return true;
+}
+
+void AsyncLeaderStepDownForReplacementTask::HandleResponse(int attempt) {
+ if (!resp_.has_error()) {
+ MarkComplete();
+ LOG_WITH_PREFIX(INFO) << Substitute(
+ "$0 succeeded (attempt $1)", type_name(), attempt);
+ return;
+ }
+
+ const Status status = StatusFromPB(resp_.error().status());
+ switch (resp_.error().code()) {
+ case TabletServerErrorPB::NOT_THE_LEADER:
+ // Not the leader anymore: goal achieved (it can now be evicted), so stop.
+ LOG_WITH_PREFIX(INFO) << Substitute(
+ "target replica $0 is no longer the leader; no further retry: $1",
+ leader_uuid_, status.ToString());
+ MarkComplete();
+ break;
+ default:
+ // Transient errors (e.g. ServiceUnavailable while a transfer is already
+ // in progress); leave the task running so it is retried until its deadline.
+ KLOG_EVERY_N_SECS(WARNING, 1) << LogPrefix() << Substitute(
+ "$0 failed with leader $1 due to error $2; will retry: $3",
+ type_name(), target_ts_desc_->ToString(),
+ TabletServerErrorPB::Code_Name(resp_.error().code()),
+ status.ToString());
+ break;
+ }
+}
+
Status CatalogManager::ProcessTabletReport(
TSDescriptor* ts_desc,
const TabletReportPB& full_report,
@@ -5794,6 +5913,14 @@ Status CatalogManager::ProcessTabletReport(
uuids_ignored_for_underreplication)) {
rpcs.emplace_back(new AsyncAddReplicaTask(
master_, tablet, cstate, RaftPeerPB::NON_VOTER, &rng_));
+ } else if (FLAGS_catalog_manager_step_down_leader_for_replacement &&
+ ShouldStepDownLeaderForReplacement(
+ config, cstate.leader_uuid(), replication_factor)) {
+ // The leader is marked with 'replace' and can't be evicted while it
+ // remains the leader; make it step down so a later report can evict
+ // it. Recovers moves whose client-driven step-down was interrupted.
+ rpcs.emplace_back(new AsyncLeaderStepDownForReplacementTask(
+ master_, tablet, cstate.leader_uuid()));
}
}
}
diff --git a/src/kudu/tools/kudu-admin-test.cc b/src/kudu/tools/kudu-admin-test.cc
index 95ee8c4bf..6fd36ca94 100644
--- a/src/kudu/tools/kudu-admin-test.cc
+++ b/src/kudu/tools/kudu-admin-test.cc
@@ -73,6 +73,7 @@
#include "kudu/mini-cluster/external_mini_cluster.h"
#include "kudu/tablet/metadata.pb.h"
#include "kudu/tablet/tablet_replica.h"
+#include "kudu/tools/tool_replica_util.h"
#include "kudu/tools/tool_test_util.h"
#include "kudu/tserver/tablet_server-test-base.h"
#include "kudu/util/cow_object.h"
@@ -491,6 +492,97 @@ INSTANTIATE_TEST_SUITE_P(EnableKudu1097AndDownTS, MoveTabletParamTest,
DownTS::TabletPeer,
DownTS::UninvolvedTS)));
+// Verify 'move_replica' is idempotent: if a previous invocation scheduled the
+// move but was interrupted before it completed (e.g. the CLI was killed),
+// re-running it resumes and completes the move instead of failing. See
+// IsReplicaMoveScheduled().
+TEST_F(AdminCliTest, TestMoveReplicaIdempotent) {
+ const auto kTimeout = MonoDelta::FromSeconds(60);
+
+ FLAGS_num_tablet_servers = 4;
+ FLAGS_num_replicas = 3;
+
+ // The recovery path is specific to the 3-4-3 replica management scheme.
+ const vector<string> kFlags = { "--raft_prepare_replacement_before_eviction=true" };
+ NO_FATALS(BuildAndStart(kFlags, kFlags));
+
+ const string master_addr = cluster_->master()->bound_rpc_addr().ToString();
+ const vector<string> master_addresses = { master_addr };
+
+ // Pick a follower as the move source and an unused tablet server as the target.
+ TServerDetails* leader = nullptr;
+ ASSERT_OK(FindTabletLeader(tablet_servers_, tablet_id_, kTimeout, &leader));
+ // Wait for the leader to commit an op in its term so that config changes
+ // (the scheduled move below) are not rejected as premature.
+ ASSERT_OK(WaitForOpFromCurrentTerm(leader, tablet_id_, COMMITTED_OPID, kTimeout));
+
+ vector<string> active_tservers;
+ for (auto it = tablet_replicas_.find(tablet_id_);
+ it != tablet_replicas_.cend() && it->first == tablet_id_; ++it) {
+ active_tservers.push_back(it->second->uuid());
+ }
+ ASSERT_EQ(FLAGS_num_replicas, active_tservers.size());
+
+ string from_ts_uuid;
+ for (const auto& uuid : active_tservers) {
+ if (uuid != leader->uuid()) {
+ from_ts_uuid = uuid;
+ break;
+ }
+ }
+ ASSERT_FALSE(from_ts_uuid.empty());
+
+ string to_ts_uuid;
+ for (const auto& entry : tablet_servers_) {
+ if (std::find(active_tservers.begin(), active_tservers.end(), entry.first) ==
+ active_tservers.end()) {
+ to_ts_uuid = entry.first;
+ break;
+ }
+ }
+ ASSERT_FALSE(to_ts_uuid.empty());
+
+ shared_ptr<KuduClient> client;
+ ASSERT_OK(KuduClientBuilder()
+ .add_master_server_addr(master_addr)
+ .Build(&client));
+
+ // Simulate an interrupted move: schedule the config change but don't wait for
+ // it to complete, mimicking a CLI process that was killed mid-move.
+ ASSERT_OK(ScheduleReplicaMove(master_addresses, client,
+ tablet_id_, from_ts_uuid, to_ts_uuid));
+
+ // The in-progress move is now detectable.
+ bool scheduled = false;
+ ASSERT_OK(IsReplicaMoveScheduled(client, tablet_id_,
+ from_ts_uuid, to_ts_uuid, &scheduled));
+ ASSERT_TRUE(scheduled);
+
+ // Re-running the same move must succeed by resuming the in-progress move.
+ // Before the idempotency fix this failed because the target replica was
+ // already present and the tablet was transiently over-replicated.
+ ASSERT_TOOL_OK(
+ "tablet",
+ "change_config",
+ "move_replica",
+ master_addr,
+ tablet_id_,
+ from_ts_uuid,
+ to_ts_uuid
+ );
+
+ // The move completed: the target hosts a voter and the source was evicted, so
+ // the config is back to the replication factor.
+ TServerDetails* to_ts = FindOrDie(tablet_servers_, to_ts_uuid);
+ ASSERT_OK(WaitUntilCommittedConfigNumVotersIs(FLAGS_num_replicas, to_ts,
+ tablet_id_, kTimeout));
+ NO_FATALS(WaitUntilCommittedConfigNumMembersIs(FLAGS_num_replicas, to_ts,
+ tablet_id_, kTimeout));
+
+ ClusterVerifier v(cluster_.get());
+ NO_FATALS(v.CheckCluster());
+}
+
Status RunUnsafeChangeConfig(const string& tablet_id,
const string& dst_host,
const vector<string>& peer_uuid_list) {
diff --git a/src/kudu/tools/tool_action_tablet.cc b/src/kudu/tools/tool_action_tablet.cc
index 6cc53c262..e406b654e 100644
--- a/src/kudu/tools/tool_action_tablet.cc
+++ b/src/kudu/tools/tool_action_tablet.cc
@@ -284,17 +284,31 @@ Status MoveReplica(const RunnerContext& context) {
const string& from_ts_uuid = FindOrDie(context.required_args, kFromTsUuidArg);
const string& to_ts_uuid = FindOrDie(context.required_args, kToTsUuidArg);
- // Check the tablet is in perfect health first.
- // We retry since occasionally ksck returns bad status because of transient
- // issues like leader elections.
- RETURN_NOT_OK_PREPEND(WaitForCleanKsck(master_addresses,
- tablet_id,
- MonoDelta::FromSeconds(5)),
- "ksck pre-move health check failed");
client::sp::shared_ptr<KuduClient> client;
RETURN_NOT_OK(CreateKuduClient(master_addresses, &client));
- RETURN_NOT_OK(ScheduleReplicaMove(master_addresses, client,
- tablet_id, from_ts_uuid, to_ts_uuid));
+
+ // Make the command idempotent: if a previous (interrupted) invocation already
+ // scheduled this move, the tablet is transiently over-replicated, so skip the
+ // pre-move health check and re-scheduling and just wait for it to complete.
+ bool already_scheduled = false;
+ RETURN_NOT_OK(IsReplicaMoveScheduled(client, tablet_id,
+ from_ts_uuid, to_ts_uuid,
+ &already_scheduled));
+ if (already_scheduled) {
+ cout << Substitute("Replica move of tablet $0 from $1 to $2 is already in "
+ "progress; resuming and waiting for it to complete",
+ tablet_id, from_ts_uuid, to_ts_uuid) << endl;
+ } else {
+ // Check the tablet is in perfect health first.
+ // We retry since occasionally ksck returns bad status because of transient
+ // issues like leader elections.
+ RETURN_NOT_OK_PREPEND(WaitForCleanKsck(master_addresses,
+ tablet_id,
+ MonoDelta::FromSeconds(5)),
+ "ksck pre-move health check failed");
+ RETURN_NOT_OK(ScheduleReplicaMove(master_addresses, client,
+ tablet_id, from_ts_uuid, to_ts_uuid));
+ }
const auto copy_timeout = MonoDelta::FromSeconds(FLAGS_move_copy_timeout_sec);
return WaitForMoveToComplete(master_addresses, client, tablet_id,
from_ts_uuid, to_ts_uuid, copy_timeout);
diff --git a/src/kudu/tools/tool_replica_util.cc b/src/kudu/tools/tool_replica_util.cc
index 0a0ae5a5d..b2e3ced87 100644
--- a/src/kudu/tools/tool_replica_util.cc
+++ b/src/kudu/tools/tool_replica_util.cc
@@ -628,6 +628,52 @@ Status ScheduleReplicaMove(const vector<string>& master_addresses,
return Status::OK();
}
+Status IsReplicaMoveScheduled(const client::sp::shared_ptr<client::KuduClient>& client,
+ const string& tablet_id,
+ const string& from_ts_uuid,
+ const string& to_ts_uuid,
+ bool* scheduled) {
+ DCHECK(scheduled);
+ *scheduled = false;
+
+ string leader_uuid;
+ HostPort leader_hp;
+ RETURN_NOT_OK(GetTabletLeader(client, tablet_id, &leader_uuid, &leader_hp));
+ unique_ptr<ConsensusServiceProxy> proxy;
+ RETURN_NOT_OK(BuildProxy(leader_hp.host(), leader_hp.port(), &proxy));
+
+ bool is_343_scheme;
+ ConsensusStatePB cstate;
+ RETURN_NOT_OK(GetConsensusState(proxy, tablet_id, leader_uuid,
+ client->default_admin_operation_timeout(),
+ &cstate, &is_343_scheme));
+
+ bool to_ts_uuid_in_config = false;
+ bool from_ts_uuid_in_config = false;
+ bool from_ts_uuid_has_replace = false;
+ for (const auto& peer : cstate.committed_config().peers()) {
+ if (peer.permanent_uuid() == to_ts_uuid) {
+ to_ts_uuid_in_config = true;
+ }
+ if (peer.permanent_uuid() == from_ts_uuid) {
+ from_ts_uuid_in_config = true;
+ from_ts_uuid_has_replace = peer.attrs().replace();
+ }
+ }
+
+ if (is_343_scheme) {
+ // 3-4-3: the target was added and the source marked with 'replace'. Already
+ // scheduled if the target is present and the source is gone or still marked.
+ *scheduled = to_ts_uuid_in_config &&
+ (!from_ts_uuid_in_config || from_ts_uuid_has_replace);
+ } else {
+ // 3-2-3: the target is added first, the source removed later. Already
+ // scheduled if both are present.
+ *scheduled = to_ts_uuid_in_config && from_ts_uuid_in_config;
+ }
+ return Status::OK();
+}
+
Status DoKsckForTablet(const vector<string>& master_addresses,
const string& tablet_id) {
shared_ptr<KsckCluster> cluster;
diff --git a/src/kudu/tools/tool_replica_util.h b/src/kudu/tools/tool_replica_util.h
index 492a01f53..3b6f3b75e 100644
--- a/src/kudu/tools/tool_replica_util.h
+++ b/src/kudu/tools/tool_replica_util.h
@@ -159,6 +159,19 @@ Status ScheduleReplicaMove(
const std::string& from_ts_uuid,
const std::string& to_ts_uuid);
+// Determine whether a replica move of tablet 'tablet_id' from 'from_ts_uuid' to
+// 'to_ts_uuid' has already been scheduled (i.e. the config change was applied).
+// Used to make 'move_replica' idempotent when a previous invocation was
+// interrupted. Sets the (non-null) 'scheduled' output parameter accordingly:
+// * 3-4-3: target present and source gone or still marked with 'replace'.
+// * 3-2-3: both target and source present.
+Status IsReplicaMoveScheduled(
+ const client::sp::shared_ptr<client::KuduClient>& client,
+ const std::string& tablet_id,
+ const std::string& from_ts_uuid,
+ const std::string& to_ts_uuid,
+ bool* scheduled);
+
// Run ksck for the tablet identified by 'tablet_id'.
Status DoKsckForTablet(
const std::vector<std::string>& master_addresses,