You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by aw...@apache.org on 2021/08/11 18:04:55 UTC
[kudu] branch master updated: [tools] Add a tool to recover master
data from tablet servers
This is an automated email from the ASF dual-hosted git repository.
awong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new ab0421b [tools] Add a tool to recover master data from tablet servers
ab0421b is described below
commit ab0421b13b81798548b7dc2f7acd8f7362d459c5
Author: Will Berkeley <wd...@apache.org>
AuthorDate: Thu Feb 22 23:53:32 2018 -0800
[tools] Add a tool to recover master data from tablet servers
This adds a tool that attempts to recover master metadata from
tablet servers using ListTablets, writing the metadata to a
syscatalog table that a new master process can use to make the
cluster operational.
It has several limitations. See the help description for the new tool.
Note that it should be possible to fix some limitations by
giving more master metadata to tablet servers. The advantage
of the tool as-is is that it should work on all versions of
Kudu since 1.0.
Change-Id: If29e421d466a531ebad72e281ae27e74e458f8c6
Reviewed-on: http://gerrit.cloudera.org:8080/9490
Tested-by: Kudu Jenkins
Reviewed-by: Bankim Bhavsar <ba...@cloudera.com>
---
src/kudu/common/partition.cc | 4 -
src/kudu/common/partition.h | 7 +-
src/kudu/common/schema.h | 8 +
src/kudu/master/catalog_manager.h | 4 +-
src/kudu/tools/CMakeLists.txt | 1 +
src/kudu/tools/kudu-admin-test.cc | 247 ++++++++++++++++++-
src/kudu/tools/master_rebuilder.cc | 350 +++++++++++++++++++++++++++
src/kudu/tools/master_rebuilder.h | 131 ++++++++++
src/kudu/tools/tool_action_common.cc | 19 ++
src/kudu/tools/tool_action_common.h | 9 +
src/kudu/tools/tool_action_master.cc | 85 +++++++
src/kudu/tools/tool_action_remote_replica.cc | 1 +
12 files changed, 852 insertions(+), 14 deletions(-)
diff --git a/src/kudu/common/partition.cc b/src/kudu/common/partition.cc
index 9cb06af..1a47db0 100644
--- a/src/kudu/common/partition.cc
+++ b/src/kudu/common/partition.cc
@@ -1174,10 +1174,6 @@ bool PartitionSchema::operator==(const PartitionSchema& rhs) const {
return true;
}
-bool PartitionSchema::operator!=(const PartitionSchema& rhs) const {
- return !(*this == rhs);
-}
-
// Encodes the specified primary key columns of the supplied row into the buffer.
void PartitionSchema::EncodeColumns(const ConstContiguousRow& row,
const vector<ColumnId>& column_ids,
diff --git a/src/kudu/common/partition.h b/src/kudu/common/partition.h
index fd72b69..ab03729 100644
--- a/src/kudu/common/partition.h
+++ b/src/kudu/common/partition.h
@@ -75,6 +75,9 @@ class Partition {
// Returns true iff the given partition 'rhs' is equivalent to this one.
bool operator==(const Partition& rhs) const;
+ bool operator!=(const Partition& rhs) const {
+ return !(*this == rhs);
+ }
// Serializes a partition into a protobuf message.
void ToPB(PartitionPB* pb) const;
@@ -300,7 +303,9 @@ class PartitionSchema {
bool operator==(const PartitionSchema& rhs) const;
// Returns 'true' iff the partition schema 'rhs' is not equivalent to this one.
- bool operator!=(const PartitionSchema& rhs) const;
+ bool operator!=(const PartitionSchema& rhs) const {
+ return !(*this == rhs);
+ }
// Transforms an exclusive lower bound range partition key into an inclusive
// lower bound range partition key.
diff --git a/src/kudu/common/schema.h b/src/kudu/common/schema.h
index 3371ba4..c00e66d 100644
--- a/src/kudu/common/schema.h
+++ b/src/kudu/common/schema.h
@@ -819,6 +819,14 @@ class Schema {
return true;
}
+ bool operator==(const Schema& other) const {
+ return this->Equals(other);
+ }
+
+ bool operator!=(const Schema& other) const {
+ return !(*this == other);
+ }
+
// Return true if the key projection schemas have exactly the same set of
// columns and respective types.
bool KeyEquals(const Schema& other,
diff --git a/src/kudu/master/catalog_manager.h b/src/kudu/master/catalog_manager.h
index 0ffc79d..f6c177c 100644
--- a/src/kudu/master/catalog_manager.h
+++ b/src/kudu/master/catalog_manager.h
@@ -305,7 +305,7 @@ class TableInfo : public RefCountedThreadSafe<TableInfo> {
void GetTabletsInRange(const GetTableLocationsRequestPB* req,
std::vector<scoped_refptr<TabletInfo>>* ret) const;
- // Adds all tablets to the vector in partition key sorted order.
+ // Fills the vector with all tablets in partition key sorted order.
void GetAllTablets(std::vector<scoped_refptr<TabletInfo>>* ret) const;
// Access the persistent metadata. Typically you should use
@@ -992,7 +992,7 @@ class CatalogManager : public tserver::TabletReplicaLookupIf {
const boost::optional<std::string>& dimension_label);
// Builds the TabletLocationsPB for a tablet based on the provided TabletInfo
- // and the replica type fiter specified. Populates locs_pb and returns
+ // and the replica type filter specified. Populates locs_pb and returns
// Status::OK on success.
//
// If 'ts_infos_dict' is not null, the returned locations use it as a dictionary
diff --git a/src/kudu/tools/CMakeLists.txt b/src/kudu/tools/CMakeLists.txt
index c465fed..a652bbe 100644
--- a/src/kudu/tools/CMakeLists.txt
+++ b/src/kudu/tools/CMakeLists.txt
@@ -102,6 +102,7 @@ target_link_libraries(kudu_tools_rebalance
#######################################
add_executable(kudu
+ master_rebuilder.cc
tool_action_cluster.cc
tool_action_diagnose.cc
tool_action_fs.cc
diff --git a/src/kudu/tools/kudu-admin-test.cc b/src/kudu/tools/kudu-admin-test.cc
index 6d3ed27..cc1ab10 100644
--- a/src/kudu/tools/kudu-admin-test.cc
+++ b/src/kudu/tools/kudu-admin-test.cc
@@ -53,6 +53,7 @@
#include "kudu/gutil/basictypes.h"
#include "kudu/gutil/map-util.h"
#include "kudu/gutil/stl_util.h"
+#include "kudu/gutil/strings/join.h"
#include "kudu/gutil/strings/split.h"
#include "kudu/gutil/strings/strip.h"
#include "kudu/gutil/strings/substitute.h"
@@ -83,6 +84,7 @@ class ListTabletsResponsePB;
DECLARE_int32(num_replicas);
DECLARE_int32(num_tablet_servers);
+DECLARE_string(sasl_protocol_name);
using kudu::client::KuduClient;
using kudu::client::KuduClientBuilder;
@@ -95,6 +97,7 @@ using kudu::client::KuduTable;
using kudu::client::KuduTableCreator;
using kudu::client::KuduValue;
using kudu::client::sp::shared_ptr;
+using kudu::cluster::ExternalMiniCluster;
using kudu::cluster::ExternalTabletServer;
using kudu::cluster::ExternalMiniCluster;
using kudu::cluster::ExternalMiniClusterOptions;
@@ -2897,17 +2900,247 @@ TEST_F(AdminCliTest, TestAddAndDropRangePartitionForMultipleRangeColumnsTable) {
});
}
-TEST_F(AdminCliTest, TestNonDefaultPrincipal) {
- ExternalMiniClusterOptions opts;
- opts.enable_kerberos = true;
- opts.principal = "oryx";
- cluster_.reset(new ExternalMiniCluster(std::move(opts)));
- ASSERT_OK(cluster_->Start());
+namespace {
+constexpr const char* kPrincipal = "oryx";
+
+vector<string> RebuildMasterCmd(const ExternalMiniCluster& cluster,
+ bool is_secure, bool log_to_stderr = false) {
+ vector<string> command = {
+ "master",
+ "unsafe_rebuild",
+ "-fs_data_dirs",
+ JoinStrings(cluster.master()->data_dirs(), ","),
+ "-fs_wal_dir",
+ cluster.master()->wal_dir(),
+ };
+ if (log_to_stderr) {
+ command.emplace_back("--logtostderr");
+ }
+ if (is_secure) {
+ command.emplace_back(Substitute("--sasl_protocol_name=$0", kPrincipal));
+ }
+ for (int i = 0; i < cluster.num_tablet_servers(); i++) {
+ auto* ts = cluster.tablet_server(i);
+ command.emplace_back(ts->bound_rpc_hostport().ToString());
+ }
+ return command;
+}
+
+void MakeTestTable(const string& table_name,
+ int num_rows,
+ int num_replicas,
+ ExternalMiniCluster* cluster) {
+ TestWorkload workload(cluster);
+ workload.set_table_name(table_name);
+ workload.set_num_replicas(num_replicas);
+ workload.Setup();
+ workload.Start();
+ while (workload.rows_inserted() < num_rows) {
+ SleepFor(MonoDelta::FromMilliseconds(10));
+ }
+ workload.StopAndJoin();
+}
+} // anonymous namespace
+
+// Test failing to run the CLI because the master is non-empty.
+TEST_F(AdminCliTest, TestRebuildMasterWhenNonEmpty) {
+ FLAGS_num_tablet_servers = 3;
+ NO_FATALS(BuildAndStart({}, {}, {}, /*create_table*/false));
+
+ constexpr const char* kTable = "default.foo";
+ NO_FATALS(MakeTestTable(kTable, /*num_rows*/10, /*num_replicas*/1, cluster_.get()));
+ NO_FATALS(cluster_->master()->Shutdown());
+ string stdout;
+ string stderr;
+ Status s = RunKuduTool(RebuildMasterCmd(*cluster_, /*is_secure*/false, /*log_to_stderr*/true),
+ &stdout, &stderr);
+ ASSERT_TRUE(s.IsRuntimeError()) << s.ToString();
+ ASSERT_STR_CONTAINS(stderr, "must be empty");
+
+ // Delete the contents of the old master from disk. This should allow the
+ // tool to run.
+ ASSERT_OK(cluster_->master()->DeleteFromDisk());
+ ASSERT_OK(RunKuduTool(RebuildMasterCmd(*cluster_, /*is_secure*/false, /*log_to_stderr*/true),
+ &stdout, &stderr));
+ ASSERT_STR_NOT_CONTAINS(stderr, "must be empty");
+ ASSERT_STR_CONTAINS(stdout,
+ "Rebuilt from 3 tablet servers, of which 0 had errors");
+ ASSERT_STR_CONTAINS(stdout, "Rebuilt from 1 replicas, of which 0 had errors");
+}
+
+// Test that the master rebuilder ignores tombstones.
+TEST_F(AdminCliTest, TestRebuildMasterWithTombstones) {
+ FLAGS_num_tablet_servers = 3;
+ NO_FATALS(BuildAndStart({}, {}, {}, /*create_table*/false));
+
+ constexpr const char* kTable = "default.foo";
+ const MonoDelta kTimeout = MonoDelta::FromSeconds(10);
+ NO_FATALS(MakeTestTable(kTable, /*num_rows*/10, /*num_replicas*/1, cluster_.get()));
+ TabletServerMap ts_map;
+ ASSERT_OK(CreateTabletServerMap(cluster_->master_proxy(),
+ cluster_->messenger(),
+ &ts_map));
+ ValueDeleter deleter(&ts_map);
+ // Find the single tablet replica and tombstone it.
+ string tablet_id;
+ TServerDetails* ts_details;
+ for (const auto& id_and_details : ts_map) {
+ vector<string> tablet_ids;
+ ts_details = id_and_details.second;
+ ASSERT_OK(ListRunningTabletIds(ts_details, kTimeout, &tablet_ids));
+ if (!tablet_ids.empty()) {
+ tablet_id = tablet_ids[0];
+ break;
+ }
+ }
+ ASSERT_FALSE(tablet_id.empty());
+ ASSERT_OK(itest::DeleteTablet(ts_details, tablet_id, tablet::TABLET_DATA_TOMBSTONED, kTimeout));
+ ASSERT_OK(itest::WaitUntilTabletInState(
+ ts_details, tablet_id, tablet::TabletStatePB::STOPPED, kTimeout));
+
+ // Rebuild the master. The tombstone shouldn't be rebuilt.
+ NO_FATALS(cluster_->master()->Shutdown());
+ ASSERT_OK(cluster_->master()->DeleteFromDisk());
+ string stdout;
+ string stderr;
+ ASSERT_OK(RunKuduTool(RebuildMasterCmd(*cluster_, /*is_secure*/false, /*log_to_stderr*/true),
+ &stdout, &stderr));
+ ASSERT_STR_CONTAINS(stderr, Substitute("Skipping replica of tablet $0 of table $1",
+ tablet_id, kTable));
+ ASSERT_STR_CONTAINS(stdout,
+ "Rebuilt from 3 tablet servers, of which 0 had errors");
+ ASSERT_STR_CONTAINS(stdout, "Rebuilt from 0 replicas, of which 0 had errors");
+
+
+ for (int i = 0; i < cluster_->num_tablet_servers(); i++) {
+ cluster_->tablet_server(i)->Shutdown();
+ }
+ ASSERT_OK(cluster_->Restart());
+
+ // Clients shouldn't be able to access the table -- it wasn't rebuilt.
+ KuduClientBuilder builder;
+ ASSERT_OK(cluster_->CreateClient(&builder, &client_));
+ client::sp::shared_ptr<KuduTable> table;
+ Status s = client_->OpenTable(kTable, &table);
+ ASSERT_TRUE(s.IsNotFound()) << s.ToString();
+
+ // We should still be able to create a table of the same name though.
+ NO_FATALS(MakeTestTable(kTable, /*num_rows*/10, /*num_replicas*/1, cluster_.get()));
+ ASSERT_OK(client_->OpenTable(kTable, &table));
+
+ ClusterVerifier cv(cluster_.get());
+ NO_FATALS(cv.CheckCluster());
+}
+
+class SecureClusterAdminCliTest : public KuduTest {
+ public:
+ void SetUpCluster(bool is_secure) {
+ ExternalMiniClusterOptions opts;
+ opts.num_tablet_servers = 3;
+ if (is_secure) {
+ opts.enable_kerberos = true;
+ opts.principal = "oryx";
+ }
+ cluster_.reset(new ExternalMiniCluster(std::move(opts)));
+ ASSERT_OK(cluster_->Start());
+ KuduClientBuilder builder;
+ builder.sasl_protocol_name(kPrincipal);
+ ASSERT_OK(cluster_->CreateClient(&builder, &client_));
+ }
+
+ void SetUp() override {
+ NO_FATALS(SetUpCluster(/*is_secure*/true));
+ }
+ protected:
+ unique_ptr<ExternalMiniCluster> cluster_;
+ client::sp::shared_ptr<KuduClient> client_;
+};
+
+TEST_F(SecureClusterAdminCliTest, TestNonDefaultPrincipal) {
ASSERT_OK(RunKuduTool({"master",
"list",
- "--sasl_protocol_name=oryx",
+ Substitute("--sasl_protocol_name=$0", kPrincipal),
HostPort::ToCommaSeparatedString(cluster_->master_rpc_addrs())}));
}
+class SecureClusterAdminCliParamTest : public SecureClusterAdminCliTest,
+ public ::testing::WithParamInterface<bool> {
+ public:
+ void SetUp() override {
+ SetUpCluster(/*is_secure*/GetParam());
+ }
+};
+
+// Basic test that the master rebuilder works in the happy case.
+TEST_P(SecureClusterAdminCliParamTest, TestRebuildMaster) {
+ bool is_secure = GetParam();
+ constexpr const char* kPreRebuildTableName = "pre_rebuild";
+ constexpr const char* kPostRebuildTableName = "post_rebuild";
+ constexpr int kNumRows = 10000;
+
+ FLAGS_num_tablet_servers = 3;
+ FLAGS_num_replicas = 3;
+
+ // Create a table and insert some rows
+ NO_FATALS(MakeTestTable(kPreRebuildTableName, kNumRows, 3, cluster_.get()));
+
+ // Scan the table to strings so we can check it isn't corrupted post-rebuild.
+ vector<string> rows;
+ {
+ client::sp::shared_ptr<KuduTable> table;
+ ASSERT_OK(client_->OpenTable(kPreRebuildTableName, &table));
+ ScanTableToStrings(table.get(), &rows);
+ }
+
+ // Shut down the master and wipe out its data.
+ NO_FATALS(cluster_->master()->Shutdown());
+ ASSERT_OK(cluster_->master()->DeleteFromDisk());
+
+ // Rebuild the master with the tool.
+ string stdout;
+ ASSERT_OK(RunKuduTool(RebuildMasterCmd(*cluster_, is_secure), &stdout));
+ ASSERT_STR_CONTAINS(stdout,
+ "Rebuilt from 3 tablet servers, of which 0 had errors");
+ ASSERT_STR_CONTAINS(stdout, "Rebuilt from 3 replicas, of which 0 had errors");
+
+ // Restart the master and the tablet servers.
+ // The tablet servers must be restarted so they accept the new master's certs.
+ for (int i = 0; i < cluster_->num_tablet_servers(); i++) {
+ cluster_->tablet_server(i)->Shutdown();
+ }
+ ASSERT_OK(cluster_->Restart());
+
+ // Verify we can read back exactly what we read before the rebuild.
+ // The client has to be rebuilt since there's a new master.
+ KuduClientBuilder builder;
+ ASSERT_OK(cluster_->CreateClient(&builder, &client_));
+ vector<string> postrebuild_rows;
+ {
+ client::sp::shared_ptr<KuduTable> table;
+ ASSERT_OK(client_->OpenTable(kPreRebuildTableName, &table));
+ ScanTableToStrings(table.get(), &postrebuild_rows);
+ }
+ ASSERT_EQ(rows.size(), postrebuild_rows.size());
+ for (int i = 0; i < rows.size(); i++) {
+ ASSERT_EQ(rows[i], postrebuild_rows[i]) << "Mismatch at row " << i
+ << " of " << rows.size();
+ }
+ // The cluster should still be considered healthy.
+ FLAGS_sasl_protocol_name = kPrincipal;
+ ClusterVerifier cv(cluster_.get());
+ NO_FATALS(cv.CheckCluster());
+
+ // Make sure we can delete the table we created before the rebuild.
+ ASSERT_OK(client_->DeleteTable(kPreRebuildTableName));
+
+ // Make sure we can create a table and write to it.
+ NO_FATALS(MakeTestTable(kPostRebuildTableName, kNumRows, 3, cluster_.get()));
+ NO_FATALS(cv.CheckCluster());
+}
+
+INSTANTIATE_TEST_SUITE_P(IsSecure, SecureClusterAdminCliParamTest,
+ ::testing::Bool());
+
+
} // namespace tools
} // namespace kudu
diff --git a/src/kudu/tools/master_rebuilder.cc b/src/kudu/tools/master_rebuilder.cc
new file mode 100644
index 0000000..a5fffa7
--- /dev/null
+++ b/src/kudu/tools/master_rebuilder.cc
@@ -0,0 +1,350 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "kudu/tools/master_rebuilder.h"
+
+#include <cstdint>
+#include <functional>
+#include <limits>
+#include <map>
+#include <memory>
+#include <ostream>
+#include <string>
+#include <vector>
+
+#include <gflags/gflags_declare.h>
+#include <glog/logging.h>
+
+#include "kudu/common/common.pb.h"
+#include "kudu/common/partition.h"
+#include "kudu/common/schema.h"
+#include "kudu/common/wire_protocol.h"
+#include "kudu/consensus/metadata.pb.h"
+#include "kudu/consensus/opid_util.h"
+#include "kudu/consensus/raft_consensus.h"
+#include "kudu/gutil/map-util.h"
+#include "kudu/gutil/ref_counted.h"
+#include "kudu/gutil/strings/substitute.h"
+#include "kudu/master/catalog_manager.h"
+#include "kudu/master/master.h"
+#include "kudu/master/master.pb.h"
+#include "kudu/master/master_options.h"
+#include "kudu/master/sys_catalog.h"
+#include "kudu/tablet/metadata.pb.h"
+#include "kudu/tablet/tablet.pb.h"
+#include "kudu/tablet/tablet_replica.h"
+#include "kudu/tools/tool_action_common.h"
+#include "kudu/tserver/tablet_server.h"
+#include "kudu/tserver/tserver_service.proxy.h"
+#include "kudu/util/cow_object.h"
+#include "kudu/util/monotime.h"
+#include "kudu/util/oid_generator.h"
+#include "kudu/util/scoped_cleanup.h"
+#include "kudu/util/status.h"
+
+DECLARE_int32(default_num_replicas);
+
+using kudu::master::Master;
+using kudu::master::MasterOptions;
+using kudu::master::SysCatalogTable;
+using kudu::master::SysTablesEntryPB;
+using kudu::master::SysTabletsEntryPB;
+using kudu::master::TableInfo;
+using kudu::master::TableMetadataLock;
+using kudu::master::TabletInfo;
+using kudu::master::TabletMetadataGroupLock;
+using kudu::master::TabletMetadataLock;
+using kudu::tserver::ListTabletsResponsePB;
+using std::string;
+using std::vector;
+using strings::Substitute;
+
+namespace kudu {
+namespace tools {
+
+namespace {
+Status NoOpCb() {
+ return Status::OK();
+}
+} // anonymous namespace
+
+MasterRebuilder::MasterRebuilder(vector<string> tserver_addrs)
+ : state_(State::NOT_DONE),
+ tserver_addrs_(std::move(tserver_addrs)) {
+}
+
+const RebuildReport& MasterRebuilder::GetRebuildReport() const {
+ CHECK_EQ(State::DONE, state_);
+ return rebuild_report_;
+}
+
+Status MasterRebuilder::RebuildMaster() {
+ CHECK_EQ(State::NOT_DONE, state_);
+
+ int bad_tservers = 0;
+ for (const auto& tserver_addr : tserver_addrs_) {
+ std::unique_ptr<tserver::TabletServerServiceProxy> proxy;
+ vector<ListTabletsResponsePB::StatusAndSchemaPB> replicas;
+ Status s = BuildProxy(tserver_addr, tserver::TabletServer::kDefaultPort, &proxy).AndThen([&]() {
+ return GetReplicas(proxy.get(), &replicas);
+ });
+ rebuild_report_.tservers.emplace_back(tserver_addr, s);
+ if (!s.ok()) {
+ LOG(WARNING) << Substitute("Failed to gather metadata from tablet server $0: $1",
+ tserver_addr, s.ToString());
+ bad_tservers++;
+ continue;
+ }
+
+ for (const auto& replica : replicas) {
+ const auto& tablet_status_pb = replica.tablet_status();
+ const auto& state_pb = tablet_status_pb.state();
+ const auto& state_str = TabletStatePB_Name(state_pb);
+ const auto& tablet_id = tablet_status_pb.tablet_id();
+ const auto& table_name = tablet_status_pb.table_name();
+ switch (state_pb) {
+ case tablet::STOPPING:
+ case tablet::STOPPED:
+ case tablet::SHUTDOWN:
+ case tablet::FAILED:
+ LOG(INFO) << Substitute("Skipping replica of tablet $0 of table $1 on tablet "
+ "server $2 in state $3", tablet_id, table_name, tserver_addr,
+ state_str);
+ continue;
+ default:
+ break;
+ }
+ Status s = CheckTableAndTabletConsistency(replica);
+ if (!s.ok()) {
+ LOG(WARNING) << Substitute("Failed to process metadata for replica of tablet $0 "
+ "of table $1 on tablet server $2 in state $3: $4",
+ tablet_id, table_name, tserver_addr, state_str, s.ToString());
+ }
+ InsertOrDieNoPrint(&rebuild_report_.replicas,
+ std::make_tuple(table_name, tablet_id, tserver_addr), s);
+ }
+ }
+
+ // Check how many tablet servers we got metadata from. We can still continue
+ // as long as one reported. If not all tablet servers returned info, our
+ // reconstructed syscatalog might be missing tables and tablets, or it might
+ // have everything.
+ if (bad_tservers > 0) {
+ LOG(WARNING) << Substitute("Failed to gather metadata from all tablet servers: "
+ "$0 of $1 tablet server(s) had errors",
+ bad_tservers, tserver_addrs_.size());
+ }
+ if (bad_tservers == tserver_addrs_.size()) {
+ return Status::ServiceUnavailable("unable to gather any tablet server metadata");
+ }
+
+ // Now that we've assembled all the metadata, we can write to a syscatalog table.
+ RETURN_NOT_OK(WriteSysCatalog());
+
+ state_ = State::DONE;
+ return Status::OK();
+}
+
+Status MasterRebuilder::CheckTableAndTabletConsistency(
+ const ListTabletsResponsePB::StatusAndSchemaPB& replica) {
+ const string& table_name = replica.tablet_status().table_name();
+ const string& tablet_id = replica.tablet_status().tablet_id();
+
+ if (!ContainsKey(tables_by_name_, table_name)) {
+ CreateTable(replica);
+ } else {
+ RETURN_NOT_OK(CheckTableConsistency(replica));
+ }
+
+ if (!ContainsKey(tablets_by_id_, tablet_id)) {
+ CreateTablet(replica);
+ }
+ return CheckTabletConsistency(replica);
+}
+
+void MasterRebuilder::CreateTable(const ListTabletsResponsePB::StatusAndSchemaPB& replica) {
+ scoped_refptr<TableInfo> table(new TableInfo(oid_generator_.Next()));
+ table->mutable_metadata()->StartMutation();
+ SysTablesEntryPB* table_metadata = &table->mutable_metadata()->mutable_dirty()->pb;
+ const string& table_name = replica.tablet_status().table_name();
+ table_metadata->set_name(table_name);
+
+ // We can't tell the schema version from ListTablets.
+ // If there's multiple schemas reported by the replicas (i.e. if an alter
+ // table is in progress), we'll fail to recover some replicas and only
+ // partially recover the table.
+ table_metadata->set_version(0);
+
+ // We can't tell the replication factor from ListTablets.
+ // We'll guess the default replication factor because it's safe and almost
+ // always correct.
+ // TODO(awong): there's probably a better heuristic based on the number of
+ // replicas reported by the tablet servers.
+ table_metadata->set_num_replicas(FLAGS_default_num_replicas);
+ table_metadata->mutable_schema()->CopyFrom(replica.schema());
+ table_metadata->mutable_partition_schema()->CopyFrom(replica.partition_schema());
+
+ // We can't tell what the next column id should be, so we guess something so
+ // large we're almost certainly OK.
+ table_metadata->set_next_column_id(std::numeric_limits<int32_t>::max() / 2);
+ table_metadata->set_state(SysTablesEntryPB::RUNNING);
+ table_metadata->set_state_msg("reconstructed by MasterRebuilder");
+ table->mutable_metadata()->CommitMutation();
+ InsertOrDie(&tables_by_name_, table_name, table);
+}
+
+Status MasterRebuilder::CheckTableConsistency(
+ const ListTabletsResponsePB::StatusAndSchemaPB& replica) {
+ const string& tablet_id = replica.tablet_status().tablet_id();
+ const string& table_name = replica.tablet_status().table_name();
+ scoped_refptr<TableInfo> table = FindOrDie(tables_by_name_, table_name);
+
+ TableMetadataLock l_table(table.get(), LockMode::READ);
+ const SysTablesEntryPB& metadata = table->metadata().state().pb;
+
+ // Check the schemas match.
+ Schema schema_from_table;
+ Schema schema_from_replica;
+ RETURN_NOT_OK(SchemaFromPB(metadata.schema(), &schema_from_table));
+ RETURN_NOT_OK(SchemaFromPB(replica.schema(), &schema_from_replica));
+ if (schema_from_table != schema_from_replica) {
+ LOG(WARNING) << Substitute("Schema mismatch for tablet $0 of table $1", tablet_id, table_name);
+ LOG(WARNING) << Substitute("Table schema: $0", schema_from_table.ToString());
+ LOG(WARNING) << Substitute("Mismatched schema: $0", schema_from_replica.ToString());
+ return Status::Corruption("inconsistent replica: schema mismatch");
+ }
+
+ // Check the partition schemas match.
+ PartitionSchema pschema_from_table;
+ PartitionSchema pschema_from_replica;
+ RETURN_NOT_OK(PartitionSchema::FromPB(metadata.partition_schema(),
+ schema_from_table,
+ &pschema_from_table));
+ RETURN_NOT_OK(PartitionSchema::FromPB(replica.partition_schema(),
+ schema_from_replica,
+ &pschema_from_replica));
+ if (pschema_from_table != pschema_from_replica) {
+ LOG(WARNING) << Substitute("Partition schema mismatch for tablet $0 of table $1",
+ tablet_id, table_name);
+ LOG(WARNING) << Substitute("First seen partition schema: $0",
+ pschema_from_table.DebugString(schema_from_table));
+ LOG(WARNING) << Substitute("Mismatched partition schema $0",
+ pschema_from_replica.DebugString(schema_from_replica));
+ return Status::Corruption("inconsistent replica: partition schema mismatch");
+ }
+ return Status::OK();
+}
+
+void MasterRebuilder::CreateTablet(const ListTabletsResponsePB::StatusAndSchemaPB& replica) {
+ const string& table_name = replica.tablet_status().table_name();
+ const string& tablet_id = replica.tablet_status().tablet_id();
+ scoped_refptr<TableInfo> table = FindOrDie(tables_by_name_, table_name);
+ scoped_refptr<TabletInfo> tablet(new TabletInfo(table, tablet_id));
+
+ tablet->mutable_metadata()->StartMutation();
+ SysTabletsEntryPB* metadata = &tablet->mutable_metadata()->mutable_dirty()->pb;
+ metadata->mutable_partition()->CopyFrom(replica.tablet_status().partition());
+
+ // Setting the term to the minimum and an invalid opid ensures that, when a
+ // master loads the reconstructed syscatalog table and receives tablet
+ // reports, it will adopt the consensus state from the reports.
+ metadata->mutable_consensus_state()->set_current_term(consensus::kMinimumTerm);
+ metadata->mutable_consensus_state()->mutable_committed_config()
+ ->set_opid_index(consensus::kInvalidOpIdIndex);
+ metadata->set_state(SysTabletsEntryPB::RUNNING);
+ metadata->set_state_msg("reconstructed by MasterRebuilder");
+ metadata->set_table_id(table->id());
+ tablet->mutable_metadata()->CommitMutation();
+ InsertOrDie(&tablets_by_id_, tablet_id, tablet);
+
+ TableMetadataLock l_table(table.get(), LockMode::WRITE);
+ TabletMetadataLock l_tablet(tablet.get(), LockMode::READ);
+ table->AddRemoveTablets({ tablet }, {});
+}
+
+Status MasterRebuilder::CheckTabletConsistency(
+ const ListTabletsResponsePB::StatusAndSchemaPB& replica) {
+ const string& tablet_id = replica.tablet_status().tablet_id();
+ const string& table_name = replica.tablet_status().table_name();
+ scoped_refptr<TabletInfo> tablet = FindOrDie(tablets_by_id_, tablet_id);
+
+ TabletMetadataLock l_tablet(tablet.get(), LockMode::READ);
+ const SysTabletsEntryPB& metadata = tablet->metadata().state().pb;
+
+ // Check the partitions match.
+ // We do not check the schemas and partition schemas match because they are
+ // already checked against the table's.
+ Partition partition_from_tablet;
+ Partition partition_from_replica;
+ Partition::FromPB(metadata.partition(), &partition_from_tablet);
+ Partition::FromPB(replica.tablet_status().partition(), &partition_from_replica);
+ if (partition_from_tablet != partition_from_replica) {
+ LOG(WARNING) << Substitute("Partition mismatch for tablet $0 of table $1",
+ tablet_id, table_name);
+ LOG(WARNING) << Substitute("First seen partition: $0",
+ metadata.partition().DebugString());
+ LOG(WARNING) << Substitute("Mismatched partition $0",
+ replica.tablet_status().partition().DebugString());;
+ return Status::Corruption("inconsistent replica: partition mismatch");
+ }
+
+ return Status::OK();
+}
+
+Status MasterRebuilder::WriteSysCatalog() {
+ // Start up the master and syscatalog.
+ MasterOptions opts;
+ Master master(opts);
+ RETURN_NOT_OK(master.Init());
+ SysCatalogTable sys_catalog(&master, &NoOpCb);
+ SCOPED_CLEANUP({
+ sys_catalog.Shutdown();
+ master.Shutdown();
+ });
+ // TODO(awong): we could be smarter about cleaning up on failure, either by
+ // initially placing data into some temp directories and moving at the end,
+ // or by deleting all blocks, WALs, and metadata on error.
+ Status s = sys_catalog.CreateNew(master.fs_manager());
+ if (s.IsAlreadyPresent()) {
+ // Adjust the error message to be clearer when running on non-empty dirs.
+ s = s.CloneAndPrepend("the specified fs directories must be empty");
+ }
+ RETURN_NOT_OK(s);
+
+ // Table-by-table, organize the metadata and write it to the syscatalog.
+ vector<scoped_refptr<TabletInfo>> tablets;
+ const auto kLeaderTimeout = MonoDelta::FromSeconds(10);
+ RETURN_NOT_OK(sys_catalog.tablet_replica()->consensus()->WaitUntilLeader(kLeaderTimeout));
+ for (const auto& table_entry : tables_by_name_) {
+ const auto& table = table_entry.second;
+ table->GetAllTablets(&tablets);
+ TableMetadataLock l_table(table.get(), LockMode::WRITE);
+ TabletMetadataGroupLock l_tablets(LockMode::RELEASED);
+ l_tablets.AddMutableInfos(tablets);
+ l_tablets.Lock(LockMode::WRITE);
+ SysCatalogTable::Actions actions;
+ actions.table_to_add = table;
+ actions.tablets_to_add = tablets;
+ RETURN_NOT_OK_PREPEND(sys_catalog.Write(actions),
+ Substitute("unable to write metadata for table $0 to sys_catalog",
+ table_entry.first));
+ }
+ return Status::OK();
+}
+
+} // namespace tools
+} // namespace kudu
diff --git a/src/kudu/tools/master_rebuilder.h b/src/kudu/tools/master_rebuilder.h
new file mode 100644
index 0000000..6c95483
--- /dev/null
+++ b/src/kudu/tools/master_rebuilder.h
@@ -0,0 +1,131 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+
+#include <map>
+#include <string>
+#include <tuple>
+#include <utility>
+#include <vector>
+
+#include "kudu/gutil/macros.h"
+#include "kudu/gutil/ref_counted.h"
+#include "kudu/master/catalog_manager.h" // IWYU pragma: keep
+#include "kudu/tserver/tserver.pb.h"
+#include "kudu/util/oid_generator.h"
+#include "kudu/util/status.h"
+
+namespace kudu {
+namespace tools {
+
+// Object for accumulating information about the rebuilding process.
+struct RebuildReport {
+ // List of (address, status) for each tablet server contacted.
+ std::vector<std::pair<std::string, Status>> tservers;
+
+ // Map (table name, tablet id, tserver address) -> status for each replica processed.
+ std::map<std::tuple<std::string, std::string, std::string>, Status> replicas;
+};
+
+// Class that reconstructs a syscatalog table from tablet servers using the
+// metadata returned by ListTablets.
+//
+// In many cases, it can create a syscatalog table that a master can use to
+// restore a cluster after a catastrophic failure of the previous masters.
+// It makes a best effort to restore as much as possible. It has the following
+// limitations:
+// - Security metadata like cryptographic keys are not rebuilt. Tablet servers
+// and clients must be restarted before starting the new master in order to
+// communicate with the new master.
+// - Table IDs are known only by the masters. Reconstructed tables will have
+// new IDs.
+// - If the table was in the process of being created, it may not be possible to
+// restore the table and it may be stuck in a half-created state.
+// - If a table was in the process of being deleted, it may be partially
+// restored and the delete request may need to be sent again.
+// - If an alter table was in progress when the masters were lost, it may not
+// be possible to restore the table.
+// - If all replicas of a tablet are missing, it may not be able to recover the
+// table fully. Moreover, the MasterRebuilder cannot detect that a tablet is
+// missing.
+// - It's not possible to determine the replication factor of a table from tablet
+// server metadata. The MasterRebuilder sets the replication factor of each
+// table to --default_num_replicas instead.
+// - It's not possible to determine the next column id for a table from tablet
+// server metadata. Instead, the MasterRebuilder sets the next column id to
+// a very large number to prevent potential conflict.
+// - Table metadata like comments, owners, and configurations are not stored on
+// tablet servers and are thus not restored.
+class MasterRebuilder {
+ public:
+ explicit MasterRebuilder(std::vector<std::string> tserver_addrs);
+ ~MasterRebuilder() = default;
+
+ // Returns the RebuildReport for this MasterRebuilder.
+ //
+ // RebuildMaster() must succeed before calling this method.
+ const RebuildReport& GetRebuildReport() const;
+
+ // Gathers metadata from tablet servers and builds a syscatalog table from it.
+ Status RebuildMaster();
+
+ private:
+ enum State {
+ NOT_DONE,
+ DONE,
+ };
+
+ // Examine the metadata of a tablet replica and merge it with the metadata
+ // from previously processed replicas.
+ Status CheckTableAndTabletConsistency(
+ const tserver::ListTabletsResponsePB::StatusAndSchemaPB& replica);
+
+ // Create the metadata for a new table or tablet from one its replicas.
+ void CreateTable(const tserver::ListTabletsResponsePB::StatusAndSchemaPB& replica);
+ void CreateTablet(const tserver::ListTabletsResponsePB::StatusAndSchemaPB& replica);
+
+ // Check that a replica's metadata is consistent with previously-gathered
+ // metadata for its table or tablet, returning an error if the input
+ // 'replica' is inconsistent with existing metadata collected for the same
+ // tablet or table.
+ Status CheckTableConsistency(const tserver::ListTabletsResponsePB::StatusAndSchemaPB& replica);
+ Status CheckTabletConsistency(const tserver::ListTabletsResponsePB::StatusAndSchemaPB& replica);
+
+ // Write the syscatalog table based on the collated tablet server metadata.
+ Status WriteSysCatalog();
+
+ State state_;
+
+ // Addresses of the tablet servers used for the reconstruction.
+ const std::vector<std::string> tserver_addrs_;
+
+ // Data structures used to organize metadata from tablet servers until it can
+ // be written to the syscatalog.
+ std::map<std::string, scoped_refptr<master::TableInfo>> tables_by_name_;
+ std::map<std::string, scoped_refptr<master::TabletInfo>> tablets_by_id_;
+
+ // Accumulator for the results of various rebuilding processes.
+ RebuildReport rebuild_report_;
+
+ // Generator for table ids, since table ids are lost if the masters are lost.
+ ObjectIdGenerator oid_generator_;
+
+ DISALLOW_COPY_AND_ASSIGN(MasterRebuilder);
+};
+
+} // namespace tools
+} // namespace kudu
diff --git a/src/kudu/tools/tool_action_common.cc b/src/kudu/tools/tool_action_common.cc
index 32843a9..482357b 100644
--- a/src/kudu/tools/tool_action_common.cc
+++ b/src/kudu/tools/tool_action_common.cc
@@ -215,6 +215,8 @@ using kudu::server::ServerClockResponsePB;
using kudu::server::ServerStatusPB;
using kudu::server::SetFlagRequestPB;
using kudu::server::SetFlagResponsePB;
+using kudu::tserver::ListTabletsRequestPB;
+using kudu::tserver::ListTabletsResponsePB;
using kudu::tserver::TabletServerAdminServiceProxy; // NOLINT
using kudu::tserver::TabletServerServiceProxy; // NOLINT
using kudu::tserver::WriteRequestPB;
@@ -503,6 +505,23 @@ Status GetServerStatus(const string& address, uint16_t default_port,
return Status::OK();
}
+Status GetReplicas(TabletServerServiceProxy* proxy,
+ vector<ListTabletsResponsePB::StatusAndSchemaPB>* replicas) {
+ ListTabletsRequestPB req;
+ ListTabletsResponsePB resp;
+ RpcController rpc;
+ rpc.set_timeout(MonoDelta::FromMilliseconds(FLAGS_timeout_ms));
+
+ RETURN_NOT_OK(proxy->ListTablets(req, &resp, &rpc));
+ if (resp.has_error()) {
+ return StatusFromPB(resp.error().status());
+ }
+
+ replicas->assign(resp.status_and_schema().begin(),
+ resp.status_and_schema().end());
+ return Status::OK();
+}
+
Status PrintSegment(const scoped_refptr<ReadableLogSegment>& segment) {
PrintEntryType print_type = ParsePrintType();
if (FLAGS_print_meta) {
diff --git a/src/kudu/tools/tool_action_common.h b/src/kudu/tools/tool_action_common.h
index 45f7330..562d426 100644
--- a/src/kudu/tools/tool_action_common.h
+++ b/src/kudu/tools/tool_action_common.h
@@ -29,6 +29,7 @@
#include "kudu/rpc/response_callback.h"
#include "kudu/tools/tool_action.h"
#include "kudu/util/net/net_util.h"
+#include "kudu/tserver/tserver.pb.h"
#include "kudu/util/status.h"
namespace kudu {
@@ -56,6 +57,10 @@ namespace server {
class ServerStatusPB;
} // namespace server
+namespace tserver {
+class TabletServerServiceProxy;
+} // namespace tserver
+
namespace tools {
// Constants for parameters and descriptions.
@@ -124,6 +129,10 @@ Status BuildProxy(const std::string& address,
Status GetServerStatus(const std::string& address, uint16_t default_port,
server::ServerStatusPB* status);
+
+Status GetReplicas(tserver::TabletServerServiceProxy* proxy,
+ std::vector<tserver::ListTabletsResponsePB::StatusAndSchemaPB>* replicas);
+
// Prints the contents of a WAL segment to stdout.
//
// The following gflags affect the output:
diff --git a/src/kudu/tools/tool_action_master.cc b/src/kudu/tools/tool_action_master.cc
index 0c308ef..a04e152 100644
--- a/src/kudu/tools/tool_action_master.cc
+++ b/src/kudu/tools/tool_action_master.cc
@@ -26,6 +26,7 @@
#include <memory>
#include <set>
#include <string>
+#include <tuple>
#include <unordered_map>
#include <utility>
#include <vector>
@@ -52,6 +53,7 @@
#include "kudu/rpc/rpc_controller.h"
#include "kudu/tools/ksck.h"
#include "kudu/tools/ksck_remote.h"
+#include "kudu/tools/master_rebuilder.h"
#include "kudu/tools/tool_action.h"
#include "kudu/tools/tool_action_common.h"
#include "kudu/util/env.h"
@@ -100,6 +102,7 @@ using kudu::master::RemoveMasterResponsePB;
using kudu::consensus::RaftPeerPB;
using kudu::rpc::RpcController;
using std::cout;
+using std::endl;
using std::map;
using std::set;
using std::string;
@@ -111,6 +114,10 @@ namespace kudu {
namespace tools {
namespace {
+const char* const kTabletServerAddressArg = "tserver_address";
+const char* const kTabletServerAddressDesc = "Address of a Kudu tablet server "
+ "of form 'hostname:port'. Port may be omitted if the tablet server is "
+ "bound to the default port.";
const char* const kFlagArg = "flag";
const char* const kValueArg = "value";
@@ -659,6 +666,38 @@ Status VerifyMasterAddressList(const vector<string>& master_addresses) {
return Status::OK();
}
+Status PrintRebuildReport(const RebuildReport& rebuild_report) {
+ cout << "Rebuild Report" << endl;
+ cout << "Tablet Servers" << endl;
+ DataTable tserver_table({ "address", "status" });
+ int bad_tservers = 0;
+ for (const auto& pair : rebuild_report.tservers) {
+ const Status& s = pair.second;
+ tserver_table.AddRow({ pair.first, s.ToString() });
+ if (!s.ok()) bad_tservers++;
+ }
+ RETURN_NOT_OK(tserver_table.PrintTo(cout));
+ cout << Substitute("Rebuilt from $0 tablet servers, of which $1 had errors",
+ rebuild_report.tservers.size(), bad_tservers)
+ << endl << endl;
+
+ cout << "Replicas" << endl;
+ DataTable replica_table({ "table", "tablet", "tablet server", "status" });
+ int bad_replicas = 0;
+ for (const auto& entry : rebuild_report.replicas) {
+ const auto& key = entry.first;
+ const Status& s = entry.second;
+ replica_table.AddRow({ std::get<0>(key), std::get<1>(key), std::get<2>(key), s.ToString() });
+ if (!s.ok()) bad_replicas++;
+ }
+ RETURN_NOT_OK(replica_table.PrintTo(cout));
+ cout << Substitute("Rebuilt from $0 replicas, of which $1 had errors",
+ rebuild_report.replicas.size(), bad_replicas)
+ << endl;
+
+ return Status::OK();
+}
+
Status RefreshAuthzCacheAtMaster(const string& master_address) {
unique_ptr<MasterServiceProxy> proxy;
RETURN_NOT_OK(BuildProxy(master_address, Master::kDefaultPort, &proxy));
@@ -709,6 +748,14 @@ Status RefreshAuthzCache(const RunnerContext& context) {
}
return Status::Incomplete(err_str);
}
+
+Status RebuildMaster(const RunnerContext& context) {
+ MasterRebuilder master_rebuilder(context.variadic_args);
+ RETURN_NOT_OK(master_rebuilder.RebuildMaster());
+ PrintRebuildReport(master_rebuilder.GetRebuildReport());
+ return Status::OK();
+}
+
} // anonymous namespace
unique_ptr<Mode> BuildMasterMode() {
@@ -861,6 +908,44 @@ unique_ptr<Mode> BuildMasterMode() {
builder.AddAction(std::move(remove_master));
}
+ {
+ const char* rebuild_extra_description = "Attempts to create on-disk metadata\n"
+ "that can be used by a non-replicated master to recover a Kudu cluster\n"
+ "that has permanently lost its masters. It has a number of limitations:\n"
+ " - Security metadata like cryptographic keys are not rebuilt. Tablet servers\n"
+ " and clients must be restarted before starting the new master in order to\n"
+ " communicate with the new master.\n"
+ " - Table IDs are known only by the masters. Reconstructed tables will have\n"
+ " new IDs.\n"
+ " - If a create, delete, or alter table was in progress when the masters were lost,\n"
+ " it may not be possible to restore the table.\n"
+ " - If all replicas of a tablet are missing, it may not be able to recover the\n"
+ " table fully. Moreover, the rebuild tool cannot detect that a tablet is\n"
+ " missing.\n"
+ " - It's not possible to determine the replication factor of a table from tablet\n"
+ " server metadata. The rebuild tool sets the replication factor of each\n"
+ " table to --default_num_replicas instead.\n"
+ " - It's not possible to determine the next column id for a table from tablet\n"
+ " server metadata. Instead, the rebuilt tool sets the next column id to\n"
+ " a very large number.\n"
+ " - Table metadata like comments, owners, and configurations are not stored on\n"
+ " tablet servers and are thus not restored.\n"
+ "WARNING: This tool is potentially unsafe. Only use it when there is no\n"
+ "possibility of recovering the original masters, and you know what you\n"
+ "are doing.";
+ unique_ptr<Action> unsafe_rebuild =
+ ActionBuilder("unsafe_rebuild", &RebuildMaster)
+ .Description("Rebuild a Kudu master from tablet server metadata")
+ .ExtraDescription(rebuild_extra_description)
+ .AddRequiredVariadicParameter({ kTabletServerAddressArg, kTabletServerAddressDesc })
+ .AddOptionalParameter("fs_data_dirs")
+ .AddOptionalParameter("fs_metadata_dir")
+ .AddOptionalParameter("fs_wal_dir")
+ .AddOptionalParameter("default_num_replicas")
+ .Build();
+ builder.AddAction(std::move(unsafe_rebuild));
+ }
+
return builder.Build();
}
diff --git a/src/kudu/tools/tool_action_remote_replica.cc b/src/kudu/tools/tool_action_remote_replica.cc
index 3b026d8..9aff3ee 100644
--- a/src/kudu/tools/tool_action_remote_replica.cc
+++ b/src/kudu/tools/tool_action_remote_replica.cc
@@ -175,6 +175,7 @@ constexpr const char* const kPeerUUIDsArg = "peer uuids";
constexpr const char* const kPeerUUIDsArgDesc =
"List of peer uuids to be part of new config, separated by whitespace";
+
Status GetReplicas(TabletServerServiceProxy* proxy,
vector<ListTabletsResponsePB::StatusAndSchemaPB>* replicas) {
ListTabletsRequestPB req;