You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by al...@apache.org on 2026/08/13 23:00:26 UTC

(kudu) branch master updated: [tools/master] make move of a leader replica recoverable

This is an automated email from the ASF dual-hosted git repository.

asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git


The following commit(s) were added to refs/heads/master by this push:
     new c667e360f [tools/master] make move of a leader replica recoverable
c667e360f is described below

commit c667e360fcbe6a4a10338cf690e4792b7252d449
Author: zhangyifan27 <ch...@163.com>
AuthorDate: Fri Jul 24 18:59:33 2026 +0800

    [tools/master] make move of a leader replica recoverable
    
    Moving a replica ('kudu tablet change_config move_replica' or 'kudu cluster
    rebalance') is client-driven: the source is marked 'replace' and, if it is
    the leader, the client asks it to step down so it can be evicted (Raft can't
    evict a leader). If the client is interrupted (Ctrl+C, or an aborted/timed-out
    rebalance) after the config change but before the leader steps down, the
    tablet is left over-replicated with a stuck, un-evictable leader.
    
    Add two recovery paths:
    
    * Master-side self-healing: once a leader marked 'replace' has its replacement
      promoted to voter, ProcessTabletReport() schedules an
      AsyncLeaderStepDownForReplacementTask to step it down so it can be evicted
      (see ShouldStepDownLeaderForReplacement(); guarded by
      --catalog_manager_step_down_leader_for_replacement, on by default, 3-4-3
      only). Driven by tablet state, so it recovers an interrupted move_replica or
      'cluster rebalance' with no client involved.
    
    * CLI idempotency (move_replica): IsReplicaMoveScheduled() lets MoveReplica()
      detect an already-scheduled move and wait for it, so re-running resumes
      instead of failing on the over-replicated tablet.
    
    Change-Id: Ic7b28ca4e41355bd82c76d50e137f06197b402bc
    Co-authored-by: Cursor <cu...@cursor.com>
    Reviewed-on: http://gerrit.cloudera.org:8080/24630
    Tested-by: Alexey Serbin <al...@apache.org>
    Reviewed-by: Alexey Serbin <al...@apache.org>
---
 src/kudu/consensus/quorum_util-test.cc             |  63 ++++++++++
 src/kudu/consensus/quorum_util.cc                  |  35 ++++++
 src/kudu/consensus/quorum_util.h                   |  14 +++
 .../raft_consensus_nonvoter-itest.cc               | 109 ++++++++++++++++++
 src/kudu/master/catalog_manager.cc                 | 127 +++++++++++++++++++++
 src/kudu/tools/kudu-admin-test.cc                  |  92 +++++++++++++++
 src/kudu/tools/tool_action_tablet.cc               |  32 ++++--
 src/kudu/tools/tool_replica_util.cc                |  46 ++++++++
 src/kudu/tools/tool_replica_util.h                 |  13 +++
 9 files changed, 522 insertions(+), 9 deletions(-)

diff --git a/src/kudu/consensus/quorum_util-test.cc b/src/kudu/consensus/quorum_util-test.cc
index 4422feea3..d12803869 100644
--- a/src/kudu/consensus/quorum_util-test.cc
+++ b/src/kudu/consensus/quorum_util-test.cc
@@ -1909,5 +1909,68 @@ TEST(QuorumUtilTest, ReplaceAllTabletReplicas) {
   EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));
 }
 
+// Verify ShouldStepDownLeaderForReplacement(): step down a leader marked with
+// 'replace' only once its replacement has been promoted to voter.
+TEST(QuorumUtilTest, ShouldStepDownLeaderForReplacement) {
+  constexpr auto kReplicationFactor = 3;
+
+  {
+    // No leader: nobody to step down.
+    RaftConfigPB config;
+    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
+    AddPeer(&config, "B", V, '+');
+    AddPeer(&config, "C", V, '+');
+    EXPECT_FALSE(ShouldStepDownLeaderForReplacement(config, "", kReplicationFactor));
+  }
+  {
+    // Leader marked, but no replacement added yet (only RF voters): don't step
+    // down early.
+    RaftConfigPB config;
+    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
+    AddPeer(&config, "B", V, '+');
+    AddPeer(&config, "C", V, '+');
+    EXPECT_FALSE(ShouldStepDownLeaderForReplacement(config, "A", kReplicationFactor));
+  }
+  {
+    // Replacement present but not yet promoted (still a NON_VOTER): don't step
+    // down.
+    RaftConfigPB config;
+    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
+    AddPeer(&config, "B", V, '+');
+    AddPeer(&config, "C", V, '+');
+    AddPeer(&config, "D", N, '+', {{"PROMOTE", true}});
+    EXPECT_FALSE(ShouldStepDownLeaderForReplacement(config, "A", kReplicationFactor));
+  }
+  {
+    // Replacement promoted to voter (RF+1 healthy voters): step down.
+    RaftConfigPB config;
+    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
+    AddPeer(&config, "B", V, '+');
+    AddPeer(&config, "C", V, '+');
+    AddPeer(&config, "D", V, '+');
+    EXPECT_TRUE(ShouldStepDownLeaderForReplacement(config, "A", kReplicationFactor));
+  }
+  {
+    // A non-leader is marked (RF+1 voters): don't step down; it is evicted
+    // directly instead.
+    RaftConfigPB config;
+    AddPeer(&config, "A", V, '+');
+    AddPeer(&config, "B", V, '+', {{"REPLACE", true}});
+    AddPeer(&config, "C", V, '+');
+    AddPeer(&config, "D", V, '+');
+    EXPECT_FALSE(ShouldStepDownLeaderForReplacement(config, "A", kReplicationFactor));
+    EXPECT_TRUE(ShouldEvictReplica(config, "A", kReplicationFactor));
+  }
+  {
+    // Fewer than RF healthy voters: don't step down (eviction couldn't commit).
+    RaftConfigPB config;
+    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
+    AddPeer(&config, "B", V, '+');
+    AddPeer(&config, "C", V, '-');
+    AddPeer(&config, "D", V, '-');
+    EXPECT_FALSE(ShouldStepDownLeaderForReplacement(config, "A", kReplicationFactor));
+  }
+}
+
 } // namespace consensus
 } // namespace kudu
diff --git a/src/kudu/consensus/quorum_util.cc b/src/kudu/consensus/quorum_util.cc
index 7b7524b03..555c91fa6 100644
--- a/src/kudu/consensus/quorum_util.cc
+++ b/src/kudu/consensus/quorum_util.cc
@@ -837,5 +837,40 @@ bool ShouldEvictReplica(const RaftConfigPB& config,
   return should_evict;
 }
 
+bool ShouldStepDownLeaderForReplacement(const RaftConfigPB& config,
+                                        const string& leader_uuid,
+                                        int replication_factor) {
+  if (leader_uuid.empty()) {
+    return false;
+  }
+
+  bool leader_with_replace = false;
+  int num_voters_total = 0;
+  int num_voters_healthy = 0;
+  for (const RaftPeerPB& peer : config.peers()) {
+    if (peer.member_type() != RaftPeerPB::VOTER) {
+      continue;
+    }
+    ++num_voters_total;
+    if (peer.health_report().overall_health() == HealthReportPB::HEALTHY) {
+      ++num_voters_healthy;
+    }
+    if (peer.permanent_uuid() == leader_uuid && peer.attrs().replace()) {
+      leader_with_replace = true;
+    }
+  }
+
+  // Step down only if the leader is marked with 'replace' and there are more
+  // than 'replication_factor' healthy voters. The latter ensures the
+  // replacement was already promoted to voter (so we don't step down early) and
+  // that a healthy majority remains to commit the ensuing eviction.
+  const bool should_step_down = leader_with_replace &&
+      num_voters_healthy > replication_factor &&
+      num_voters_healthy >= MajoritySize(num_voters_total);
+  VLOG(2) << "decision: should" << (should_step_down ? "" : " not")
+          << " step down leader " << leader_uuid << " for replacement";
+  return should_step_down;
+}
+
 }  // namespace consensus
 }  // namespace kudu
diff --git a/src/kudu/consensus/quorum_util.h b/src/kudu/consensus/quorum_util.h
index b78c9104a..12edd3547 100644
--- a/src/kudu/consensus/quorum_util.h
+++ b/src/kudu/consensus/quorum_util.h
@@ -124,6 +124,20 @@ bool ShouldEvictReplica(const RaftConfigPB& config,
                         int replication_factor,
                         std::string* uuid_to_evict = nullptr);
 
+// Whether the leader replica should be made to step down so that it can be
+// evicted afterwards. Kudu's Raft implementation cannot evict the leader
+// replica (see ShouldEvictReplica()), so a leader marked with the 'replace'
+// attribute must first step down. This lets the master recover when the
+// client-driven step-down (CLI or auto-rebalancer) was interrupted.
+//
+// Returns 'true' when the leader is marked with 'replace' and there are more
+// than 'replication_factor' healthy voters. The latter both keeps a healthy
+// majority for the ensuing eviction and ensures the replacement replica has
+// already been promoted to voter before the leader steps down.
+bool ShouldStepDownLeaderForReplacement(const RaftConfigPB& config,
+                                        const std::string& leader_uuid,
+                                        int replication_factor);
+
 }  // namespace consensus
 }  // namespace kudu
 
diff --git a/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc b/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc
index d81aa7bac..e29d514eb 100644
--- a/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc
+++ b/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc
@@ -86,6 +86,7 @@ using kudu::consensus::IsRaftConfigMember;
 using kudu::consensus::IsRaftConfigVoter;
 using kudu::consensus::RaftPeerPB;
 using kudu::itest::AddServer;
+using kudu::itest::BulkChangeConfig;
 using kudu::itest::GetConsensusState;
 using kudu::itest::GetInt64Metric;
 using kudu::itest::GetTableLocations;
@@ -98,6 +99,7 @@ using kudu::itest::TabletServerMap;
 using kudu::itest::WAIT_FOR_LEADER;
 using kudu::itest::WaitForNumTabletsOnTS;
 using kudu::itest::WaitForReplicasReportedToMaster;
+using kudu::itest::WaitUntilCommittedConfigNumVotersIs;
 using kudu::master::ANY_REPLICA;
 using kudu::master::GetTableLocationsResponsePB;
 using kudu::master::GetTabletLocationsResponsePB;
@@ -1246,6 +1248,113 @@ TEST_F(RaftConsensusNonVoterITest, CatalogManagerEvictsExcessNonVoter) {
   NO_FATALS(cluster_->AssertNoCrashes());
 }
 
+// With a leader marked 'replace' and no client involved (e.g. an interrupted
+// move_replica/rebalance), the master must add+promote a replacement, step the
+// leader down and evict it. With the mechanism disabled the tablet instead
+// stays stuck over-replicated. See ShouldStepDownLeaderForReplacement().
+class LeaderReplacementRecoveryITest :
+    public RaftConsensusNonVoterITest,
+    public ::testing::WithParamInterface<bool> {
+};
+
+INSTANTIATE_TEST_SUITE_P(, LeaderReplacementRecoveryITest, ::testing::Bool());
+
+TEST_P(LeaderReplacementRecoveryITest, StepDownAndEvictMarkedLeader) {
+  const bool enable_step_down = GetParam();
+  const MonoDelta kTimeout = MonoDelta::FromSeconds(60);
+  const int kReplicasNum = 3;
+  FLAGS_num_replicas = kReplicasNum;
+  // One extra tablet server so the master can add a replacement replica.
+  FLAGS_num_tablet_servers = kReplicasNum + 1;
+
+  const vector<string> kTserverFlags = {
+    "--raft_prepare_replacement_before_eviction=true",
+    // Heartbeat to the master often so it reacts within a few heartbeats; this
+    // makes the negative (disabled) case below trustworthy: if the master were
+    // going to evict the marked leader, it would have many chances to do so
+    // within the bounded wait.
+    "--heartbeat_interval_ms=100",
+  };
+  const vector<string> kMasterFlags = {
+    "--raft_prepare_replacement_before_eviction=true",
+    Substitute("--catalog_manager_step_down_leader_for_replacement=$0",
+               enable_step_down),
+  };
+  NO_FATALS(BuildAndStart(kTserverFlags, kMasterFlags));
+
+  // Committed op in the leader's term is required before changing the config.
+  TServerDetails* leader = nullptr;
+  ASSERT_OK(WaitForLeaderWithCommittedOp(tablet_id_, kTimeout, &leader));
+  const string leader_uuid = leader->uuid();
+
+  // A follower that stays in the config, used to observe the outcome.
+  TServerDetails* observer = nullptr;
+  for (const auto& e : tablet_replicas_) {
+    if (e.first == tablet_id_ && e.second->uuid() != leader_uuid) {
+      observer = e.second;
+      break;
+    }
+  }
+  ASSERT_NE(nullptr, observer);
+
+  // Mark the leader 'replace' (what an interrupted move leaves behind) without
+  // adding a replacement: the master must drive everything from here.
+  consensus::ConsensusStatePB cstate;
+  ASSERT_OK(GetConsensusState(leader, tablet_id_, kTimeout,
+                              EXCLUDE_HEALTH_REPORT, &cstate));
+  {
+    vector<consensus::BulkChangeConfigRequestPB::ConfigChangeItemPB> changes;
+    consensus::BulkChangeConfigRequestPB::ConfigChangeItemPB change;
+    change.set_type(consensus::MODIFY_PEER);
+    RaftPeerPB* peer = change.mutable_peer();
+    peer->set_permanent_uuid(leader_uuid);
+    peer->mutable_attrs()->set_replace(true);
+    changes.emplace_back(std::move(change));
+    ASSERT_OK(BulkChangeConfig(leader, tablet_id_, changes, kTimeout,
+                               cstate.committed_config().opid_index()));
+  }
+
+  if (enable_step_down) {
+    // Terminal state: the old leader is evicted and the config is back at RF.
+    ASSERT_EVENTUALLY([&] {
+      consensus::ConsensusStatePB cs;
+      ASSERT_OK(GetConsensusState(observer, tablet_id_, kTimeout,
+                                  EXCLUDE_HEALTH_REPORT, &cs));
+      ASSERT_EQ(kReplicasNum, cs.committed_config().peers_size());
+      for (const auto& peer : cs.committed_config().peers()) {
+        ASSERT_NE(leader_uuid, peer.permanent_uuid())
+            << "the 'replace'-marked leader was not evicted";
+      }
+    });
+  } else {
+    // The replacement is still added and promoted (RF+1 voters), but the marked
+    // leader can never be evicted, so the tablet stays stuck over-replicated.
+    ASSERT_OK(WaitUntilCommittedConfigNumVotersIs(kReplicasNum + 1, observer,
+                                                  tablet_id_, kTimeout));
+    if (AllowSlowTests()) {
+      // With frequent heartbeats the master gets dozens of chances to act over
+      // this window; confirm it never evicts the marked leader, which stays
+      // present and keeps leading.
+      SleepFor(MonoDelta::FromSeconds(5));
+      consensus::ConsensusStatePB cs;
+      ASSERT_OK(GetConsensusState(observer, tablet_id_, kTimeout,
+                                  EXCLUDE_HEALTH_REPORT, &cs));
+      ASSERT_EQ(kReplicasNum + 1, cs.committed_config().peers_size());
+      ASSERT_EQ(leader_uuid, cs.leader_uuid());
+      bool leader_still_present = false;
+      for (const auto& peer : cs.committed_config().peers()) {
+        if (peer.permanent_uuid() == leader_uuid) {
+          leader_still_present = true;
+          break;
+        }
+      }
+      ASSERT_TRUE(leader_still_present);
+    }
+  }
+
+  NO_FATALS(cluster_->AssertNoCrashes());
+}
+
 // Check that the catalog manager adds a non-voter replica to replace failed
 // voter replica in a tablet.
 //
diff --git a/src/kudu/master/catalog_manager.cc b/src/kudu/master/catalog_manager.cc
index d29383626..c400d59cb 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -302,6 +302,15 @@ DEFINE_bool(catalog_manager_evict_excess_replicas, true,
 TAG_FLAG(catalog_manager_evict_excess_replicas, hidden);
 TAG_FLAG(catalog_manager_evict_excess_replicas, runtime);
 
+DEFINE_bool(catalog_manager_step_down_leader_for_replacement, true,
+            "Whether the catalog manager makes a leader replica marked with the "
+            "'replace' attribute step down so it can be evicted. This lets the "
+            "master complete a move of a leader replica even if the client that "
+            "initiated it was interrupted. Only effective with the 3-4-3 replica "
+            "management scheme (--raft_prepare_replacement_before_eviction).");
+TAG_FLAG(catalog_manager_step_down_leader_for_replacement, advanced);
+TAG_FLAG(catalog_manager_step_down_leader_for_replacement, runtime);
+
 DEFINE_int32(catalog_manager_inject_latency_list_authz_ms, 0,
              "Injects a sleep in milliseconds while authorizing a ListTables "
              "request. This is a test-only flag.");
@@ -474,9 +483,13 @@ using kudu::cfile::TypeEncodingInfo;
 using kudu::consensus::ConsensusServiceProxy;
 using kudu::consensus::ConsensusStatePB;
 using kudu::consensus::IsRaftConfigMember;
+using kudu::consensus::LeaderStepDownMode;
+using kudu::consensus::LeaderStepDownRequestPB;
+using kudu::consensus::LeaderStepDownResponsePB;
 using kudu::consensus::RaftConfigPB;
 using kudu::consensus::RaftConsensus;
 using kudu::consensus::RaftPeerPB;
+using kudu::consensus::ShouldStepDownLeaderForReplacement;
 using kudu::consensus::StartTabletCopyRequestPB;
 using kudu::consensus::kMinimumTerm;
 using kudu::hms::HmsClientVerifyKuduSyncConfig;
@@ -5474,6 +5487,112 @@ bool AsyncEvictReplicaTask::SendRequest(int attempt) {
   return true;
 }
 
+// Task to make a leader replica marked with the 'replace' attribute step down
+// so that the catalog manager can evict it (Raft cannot evict a leader). See
+// ShouldStepDownLeaderForReplacement().
+//
+// The task targets the specific replica captured at creation time; if that
+// replica is no longer the leader when the RPC is sent, the task completes
+// without doing anything, so it never steps down an unrelated, healthy leader.
+class AsyncLeaderStepDownForReplacementTask : public RetryingTSRpcTask {
+ public:
+  AsyncLeaderStepDownForReplacementTask(Master* master,
+                                        scoped_refptr<TabletInfo> tablet,
+                                        string leader_uuid);
+
+  string type_name() const override;
+  string description() const override;
+
+ protected:
+  bool SendRequest(int attempt) override;
+  void HandleResponse(int attempt) override;
+
+ private:
+  const string& tablet_id() const override { return tablet_->id(); }
+
+  const scoped_refptr<TabletInfo> tablet_;
+  const string leader_uuid_;
+
+  LeaderStepDownResponsePB resp_;
+};
+
+AsyncLeaderStepDownForReplacementTask::AsyncLeaderStepDownForReplacementTask(
+    Master* master,
+    scoped_refptr<TabletInfo> tablet,
+    string leader_uuid)
+    : RetryingTSRpcTask(master,
+                        unique_ptr<TSPicker>(new PickSpecificUUID(leader_uuid)),
+                        tablet->table().get()),
+      tablet_(std::move(tablet)),
+      leader_uuid_(std::move(leader_uuid)) {
+}
+
+string AsyncLeaderStepDownForReplacementTask::type_name() const {
+  return "LeaderStepDownForReplacement";
+}
+
+string AsyncLeaderStepDownForReplacementTask::description() const {
+  return Substitute("$0 RPC for tablet $1 (leader $2)",
+                    type_name(), tablet_->id(), leader_uuid_);
+}
+
+bool AsyncLeaderStepDownForReplacementTask::SendRequest(int attempt) {
+  // If the targeted replica is no longer the leader, bail out to avoid asking a
+  // different, possibly healthy, leader to step down.
+  {
+    TabletMetadataLock l(tablet_.get(), LockMode::READ);
+    const auto& current_leader =
+        l.data().pb.consensus_state().leader_uuid();
+    if (current_leader != leader_uuid_) {
+      LOG_WITH_PREFIX(INFO) << Substitute(
+          "replica $0 is no longer the leader (current leader: '$1'); "
+          "no leader step-down needed", leader_uuid_, current_leader);
+      MarkComplete();
+      return false;
+    }
+  }
+
+  LeaderStepDownRequestPB req;
+  req.set_dest_uuid(leader_uuid_);
+  req.set_tablet_id(tablet_->id());
+  req.set_mode(LeaderStepDownMode::GRACEFUL);
+  VLOG(1) << Substitute("Sending $0 request to $1 (attempt $2): $3",
+                        type_name(), target_ts_desc_->ToString(), attempt,
+                        SecureDebugString(req));
+  consensus_proxy_->LeaderStepDownAsync(req, &resp_, &rpc_,
+                                        [this]() { this->RpcCallback(); });
+  return true;
+}
+
+void AsyncLeaderStepDownForReplacementTask::HandleResponse(int attempt) {
+  if (!resp_.has_error()) {
+    MarkComplete();
+    LOG_WITH_PREFIX(INFO) << Substitute(
+        "$0 succeeded (attempt $1)", type_name(), attempt);
+    return;
+  }
+
+  const Status status = StatusFromPB(resp_.error().status());
+  switch (resp_.error().code()) {
+    case TabletServerErrorPB::NOT_THE_LEADER:
+      // Not the leader anymore: goal achieved (it can now be evicted), so stop.
+      LOG_WITH_PREFIX(INFO) << Substitute(
+          "target replica $0 is no longer the leader; no further retry: $1",
+          leader_uuid_, status.ToString());
+      MarkComplete();
+      break;
+    default:
+      // Transient errors (e.g. ServiceUnavailable while a transfer is already
+      // in progress); leave the task running so it is retried until its deadline.
+      KLOG_EVERY_N_SECS(WARNING, 1) << LogPrefix() << Substitute(
+          "$0 failed with leader $1 due to error $2; will retry: $3",
+          type_name(), target_ts_desc_->ToString(),
+          TabletServerErrorPB::Code_Name(resp_.error().code()),
+          status.ToString());
+      break;
+  }
+}
+
 Status CatalogManager::ProcessTabletReport(
     TSDescriptor* ts_desc,
     const TabletReportPB& full_report,
@@ -5794,6 +5913,14 @@ Status CatalogManager::ProcessTabletReport(
                                     uuids_ignored_for_underreplication)) {
           rpcs.emplace_back(new AsyncAddReplicaTask(
               master_, tablet, cstate, RaftPeerPB::NON_VOTER, &rng_));
+        } else if (FLAGS_catalog_manager_step_down_leader_for_replacement &&
+                   ShouldStepDownLeaderForReplacement(
+                       config, cstate.leader_uuid(), replication_factor)) {
+          // The leader is marked with 'replace' and can't be evicted while it
+          // remains the leader; make it step down so a later report can evict
+          // it. Recovers moves whose client-driven step-down was interrupted.
+          rpcs.emplace_back(new AsyncLeaderStepDownForReplacementTask(
+              master_, tablet, cstate.leader_uuid()));
         }
       }
     }
diff --git a/src/kudu/tools/kudu-admin-test.cc b/src/kudu/tools/kudu-admin-test.cc
index 95ee8c4bf..6fd36ca94 100644
--- a/src/kudu/tools/kudu-admin-test.cc
+++ b/src/kudu/tools/kudu-admin-test.cc
@@ -73,6 +73,7 @@
 #include "kudu/mini-cluster/external_mini_cluster.h"
 #include "kudu/tablet/metadata.pb.h"
 #include "kudu/tablet/tablet_replica.h"
+#include "kudu/tools/tool_replica_util.h"
 #include "kudu/tools/tool_test_util.h"
 #include "kudu/tserver/tablet_server-test-base.h"
 #include "kudu/util/cow_object.h"
@@ -491,6 +492,97 @@ INSTANTIATE_TEST_SUITE_P(EnableKudu1097AndDownTS, MoveTabletParamTest,
                                                               DownTS::TabletPeer,
                                                               DownTS::UninvolvedTS)));
 
+// Verify 'move_replica' is idempotent: if a previous invocation scheduled the
+// move but was interrupted before it completed (e.g. the CLI was killed),
+// re-running it resumes and completes the move instead of failing. See
+// IsReplicaMoveScheduled().
+TEST_F(AdminCliTest, TestMoveReplicaIdempotent) {
+  const auto kTimeout = MonoDelta::FromSeconds(60);
+
+  FLAGS_num_tablet_servers = 4;
+  FLAGS_num_replicas = 3;
+
+  // The recovery path is specific to the 3-4-3 replica management scheme.
+  const vector<string> kFlags = { "--raft_prepare_replacement_before_eviction=true" };
+  NO_FATALS(BuildAndStart(kFlags, kFlags));
+
+  const string master_addr = cluster_->master()->bound_rpc_addr().ToString();
+  const vector<string> master_addresses = { master_addr };
+
+  // Pick a follower as the move source and an unused tablet server as the target.
+  TServerDetails* leader = nullptr;
+  ASSERT_OK(FindTabletLeader(tablet_servers_, tablet_id_, kTimeout, &leader));
+  // Wait for the leader to commit an op in its term so that config changes
+  // (the scheduled move below) are not rejected as premature.
+  ASSERT_OK(WaitForOpFromCurrentTerm(leader, tablet_id_, COMMITTED_OPID, kTimeout));
+
+  vector<string> active_tservers;
+  for (auto it = tablet_replicas_.find(tablet_id_);
+       it != tablet_replicas_.cend() && it->first == tablet_id_; ++it) {
+    active_tservers.push_back(it->second->uuid());
+  }
+  ASSERT_EQ(FLAGS_num_replicas, active_tservers.size());
+
+  string from_ts_uuid;
+  for (const auto& uuid : active_tservers) {
+    if (uuid != leader->uuid()) {
+      from_ts_uuid = uuid;
+      break;
+    }
+  }
+  ASSERT_FALSE(from_ts_uuid.empty());
+
+  string to_ts_uuid;
+  for (const auto& entry : tablet_servers_) {
+    if (std::find(active_tservers.begin(), active_tservers.end(), entry.first) ==
+        active_tservers.end()) {
+      to_ts_uuid = entry.first;
+      break;
+    }
+  }
+  ASSERT_FALSE(to_ts_uuid.empty());
+
+  shared_ptr<KuduClient> client;
+  ASSERT_OK(KuduClientBuilder()
+            .add_master_server_addr(master_addr)
+            .Build(&client));
+
+  // Simulate an interrupted move: schedule the config change but don't wait for
+  // it to complete, mimicking a CLI process that was killed mid-move.
+  ASSERT_OK(ScheduleReplicaMove(master_addresses, client,
+                                tablet_id_, from_ts_uuid, to_ts_uuid));
+
+  // The in-progress move is now detectable.
+  bool scheduled = false;
+  ASSERT_OK(IsReplicaMoveScheduled(client, tablet_id_,
+                                   from_ts_uuid, to_ts_uuid, &scheduled));
+  ASSERT_TRUE(scheduled);
+
+  // Re-running the same move must succeed by resuming the in-progress move.
+  // Before the idempotency fix this failed because the target replica was
+  // already present and the tablet was transiently over-replicated.
+  ASSERT_TOOL_OK(
+    "tablet",
+    "change_config",
+    "move_replica",
+    master_addr,
+    tablet_id_,
+    from_ts_uuid,
+    to_ts_uuid
+  );
+
+  // The move completed: the target hosts a voter and the source was evicted, so
+  // the config is back to the replication factor.
+  TServerDetails* to_ts = FindOrDie(tablet_servers_, to_ts_uuid);
+  ASSERT_OK(WaitUntilCommittedConfigNumVotersIs(FLAGS_num_replicas, to_ts,
+                                                tablet_id_, kTimeout));
+  NO_FATALS(WaitUntilCommittedConfigNumMembersIs(FLAGS_num_replicas, to_ts,
+                                                 tablet_id_, kTimeout));
+
+  ClusterVerifier v(cluster_.get());
+  NO_FATALS(v.CheckCluster());
+}
+
 Status RunUnsafeChangeConfig(const string& tablet_id,
                              const string& dst_host,
                              const vector<string>& peer_uuid_list) {
diff --git a/src/kudu/tools/tool_action_tablet.cc b/src/kudu/tools/tool_action_tablet.cc
index 6cc53c262..e406b654e 100644
--- a/src/kudu/tools/tool_action_tablet.cc
+++ b/src/kudu/tools/tool_action_tablet.cc
@@ -284,17 +284,31 @@ Status MoveReplica(const RunnerContext& context) {
   const string& from_ts_uuid = FindOrDie(context.required_args, kFromTsUuidArg);
   const string& to_ts_uuid = FindOrDie(context.required_args, kToTsUuidArg);
 
-  // Check the tablet is in perfect health first.
-  // We retry since occasionally ksck returns bad status because of transient
-  // issues like leader elections.
-  RETURN_NOT_OK_PREPEND(WaitForCleanKsck(master_addresses,
-                                         tablet_id,
-                                         MonoDelta::FromSeconds(5)),
-                        "ksck pre-move health check failed");
   client::sp::shared_ptr<KuduClient> client;
   RETURN_NOT_OK(CreateKuduClient(master_addresses, &client));
-  RETURN_NOT_OK(ScheduleReplicaMove(master_addresses, client,
-                                    tablet_id, from_ts_uuid, to_ts_uuid));
+
+  // Make the command idempotent: if a previous (interrupted) invocation already
+  // scheduled this move, the tablet is transiently over-replicated, so skip the
+  // pre-move health check and re-scheduling and just wait for it to complete.
+  bool already_scheduled = false;
+  RETURN_NOT_OK(IsReplicaMoveScheduled(client, tablet_id,
+                                       from_ts_uuid, to_ts_uuid,
+                                       &already_scheduled));
+  if (already_scheduled) {
+    cout << Substitute("Replica move of tablet $0 from $1 to $2 is already in "
+                       "progress; resuming and waiting for it to complete",
+                       tablet_id, from_ts_uuid, to_ts_uuid) << endl;
+  } else {
+    // Check the tablet is in perfect health first.
+    // We retry since occasionally ksck returns bad status because of transient
+    // issues like leader elections.
+    RETURN_NOT_OK_PREPEND(WaitForCleanKsck(master_addresses,
+                                           tablet_id,
+                                           MonoDelta::FromSeconds(5)),
+                          "ksck pre-move health check failed");
+    RETURN_NOT_OK(ScheduleReplicaMove(master_addresses, client,
+                                      tablet_id, from_ts_uuid, to_ts_uuid));
+  }
   const auto copy_timeout = MonoDelta::FromSeconds(FLAGS_move_copy_timeout_sec);
   return WaitForMoveToComplete(master_addresses, client, tablet_id,
                                from_ts_uuid, to_ts_uuid, copy_timeout);
diff --git a/src/kudu/tools/tool_replica_util.cc b/src/kudu/tools/tool_replica_util.cc
index 0a0ae5a5d..b2e3ced87 100644
--- a/src/kudu/tools/tool_replica_util.cc
+++ b/src/kudu/tools/tool_replica_util.cc
@@ -628,6 +628,52 @@ Status ScheduleReplicaMove(const vector<string>& master_addresses,
   return Status::OK();
 }
 
+Status IsReplicaMoveScheduled(const client::sp::shared_ptr<client::KuduClient>& client,
+                              const string& tablet_id,
+                              const string& from_ts_uuid,
+                              const string& to_ts_uuid,
+                              bool* scheduled) {
+  DCHECK(scheduled);
+  *scheduled = false;
+
+  string leader_uuid;
+  HostPort leader_hp;
+  RETURN_NOT_OK(GetTabletLeader(client, tablet_id, &leader_uuid, &leader_hp));
+  unique_ptr<ConsensusServiceProxy> proxy;
+  RETURN_NOT_OK(BuildProxy(leader_hp.host(), leader_hp.port(), &proxy));
+
+  bool is_343_scheme;
+  ConsensusStatePB cstate;
+  RETURN_NOT_OK(GetConsensusState(proxy, tablet_id, leader_uuid,
+                                  client->default_admin_operation_timeout(),
+                                  &cstate, &is_343_scheme));
+
+  bool to_ts_uuid_in_config = false;
+  bool from_ts_uuid_in_config = false;
+  bool from_ts_uuid_has_replace = false;
+  for (const auto& peer : cstate.committed_config().peers()) {
+    if (peer.permanent_uuid() == to_ts_uuid) {
+      to_ts_uuid_in_config = true;
+    }
+    if (peer.permanent_uuid() == from_ts_uuid) {
+      from_ts_uuid_in_config = true;
+      from_ts_uuid_has_replace = peer.attrs().replace();
+    }
+  }
+
+  if (is_343_scheme) {
+    // 3-4-3: the target was added and the source marked with 'replace'. Already
+    // scheduled if the target is present and the source is gone or still marked.
+    *scheduled = to_ts_uuid_in_config &&
+        (!from_ts_uuid_in_config || from_ts_uuid_has_replace);
+  } else {
+    // 3-2-3: the target is added first, the source removed later. Already
+    // scheduled if both are present.
+    *scheduled = to_ts_uuid_in_config && from_ts_uuid_in_config;
+  }
+  return Status::OK();
+}
+
 Status DoKsckForTablet(const vector<string>& master_addresses,
                        const string& tablet_id) {
   shared_ptr<KsckCluster> cluster;
diff --git a/src/kudu/tools/tool_replica_util.h b/src/kudu/tools/tool_replica_util.h
index 492a01f53..3b6f3b75e 100644
--- a/src/kudu/tools/tool_replica_util.h
+++ b/src/kudu/tools/tool_replica_util.h
@@ -159,6 +159,19 @@ Status ScheduleReplicaMove(
     const std::string& from_ts_uuid,
     const std::string& to_ts_uuid);
 
+// Determine whether a replica move of tablet 'tablet_id' from 'from_ts_uuid' to
+// 'to_ts_uuid' has already been scheduled (i.e. the config change was applied).
+// Used to make 'move_replica' idempotent when a previous invocation was
+// interrupted. Sets the (non-null) 'scheduled' output parameter accordingly:
+//   * 3-4-3: target present and source gone or still marked with 'replace'.
+//   * 3-2-3: both target and source present.
+Status IsReplicaMoveScheduled(
+    const client::sp::shared_ptr<client::KuduClient>& client,
+    const std::string& tablet_id,
+    const std::string& from_ts_uuid,
+    const std::string& to_ts_uuid,
+    bool* scheduled);
+
 // Run ksck for the tablet identified by 'tablet_id'.
 Status DoKsckForTablet(
     const std::vector<std::string>& master_addresses,