You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by to...@apache.org on 2016/01/26 21:49:15 UTC
incubator-kudu git commit: KUDU-1054 (Part 1). Failover client scans
on TABLET_NOT_FOUND
Repository: incubator-kudu
Updated Branches:
refs/heads/master f09025432 -> 1204c0a9a
KUDU-1054 (Part 1). Failover client scans on TABLET_NOT_FOUND
If a tablet is deleted from a tablet server, a client that attempts a
scan on the tablet should fail over to another replica if possible.
In addition, this patch allows for marking a tablet as stale in the meta
cache. It replaces RemoteTablet::InvalidateCachedReplicas() with a
MarkStale() method, since InvalidateCachedReplicas() was destructive and
made it difficult to use in a multithreading context.
By adding a "stale" flag to RemoteTablet, we allow users to access
stale replicas until they are replaced (under a lock), which simplifies
the multithreading behavior.
Added a test for failing over client scans that uses tablet deletion and
configuration change to cause a new node to become leader, which
requires updating the tablet locations to scan from.
Also added some test-related helper functions.
Change-Id: Ic97714d82bd943beeb0350babc888b7e5c14ed57
Reviewed-on: http://gerrit.cloudera.org:8080/977
Reviewed-by: Todd Lipcon <to...@apache.org>
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/1204c0a9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/1204c0a9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/1204c0a9
Branch: refs/heads/master
Commit: 1204c0a9afb28687e8b2312429d71fa8589cc62f
Parents: f090254
Author: Mike Percy <mp...@apache.org>
Authored: Fri Jan 22 13:33:30 2016 -0800
Committer: Todd Lipcon <to...@apache.org>
Committed: Tue Jan 26 20:46:14 2016 +0000
----------------------------------------------------------------------
src/kudu/client/client-test-util.cc | 7 +-
src/kudu/client/client-test-util.h | 4 +
src/kudu/client/client-test.cc | 2 +-
src/kudu/client/meta_cache.cc | 17 ++-
src/kudu/client/meta_cache.h | 21 ++-
src/kudu/client/scanner-internal.cc | 54 ++++---
src/kudu/integration-tests/CMakeLists.txt | 1 +
.../integration-tests/client_failover-itest.cc | 151 +++++++++++++++++++
.../external_mini_cluster_fs_inspector.cc | 10 ++
.../external_mini_cluster_fs_inspector.h | 1 +
10 files changed, 238 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/1204c0a9/src/kudu/client/client-test-util.cc
----------------------------------------------------------------------
diff --git a/src/kudu/client/client-test-util.cc b/src/kudu/client/client-test-util.cc
index 7015c8b..631b735 100644
--- a/src/kudu/client/client-test-util.cc
+++ b/src/kudu/client/client-test-util.cc
@@ -16,7 +16,6 @@
// under the License.
#include "kudu/client/client-test-util.h"
-#include "kudu/client/row_result.h"
#include <vector>
@@ -57,6 +56,12 @@ void ScanTableToStrings(KuduTable* table, vector<string>* row_strings) {
ScanToStrings(&scanner, row_strings);
}
+int64_t CountTableRows(KuduTable* table) {
+ vector<string> rows;
+ client::ScanTableToStrings(table, &rows);
+ return rows.size();
+}
+
void ScanToStrings(KuduScanner* scanner, vector<string>* row_strings) {
ASSERT_OK(scanner->Open());
vector<KuduRowResult> rows;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/1204c0a9/src/kudu/client/client-test-util.h
----------------------------------------------------------------------
diff --git a/src/kudu/client/client-test-util.h b/src/kudu/client/client-test-util.h
index 2932154..882b9c4 100644
--- a/src/kudu/client/client-test-util.h
+++ b/src/kudu/client/client-test-util.h
@@ -44,8 +44,12 @@ inline void FlushSessionOrDie(const sp::shared_ptr<KuduSession>& session) {
}
}
+// Scans in LEADER_ONLY mode, returning stringified rows in the given vector.
void ScanTableToStrings(KuduTable* table, std::vector<std::string>* row_strings);
+// Count the number of rows in the table in LEADER_ONLY mode.
+int64_t CountTableRows(KuduTable* table);
+
void ScanToStrings(KuduScanner* scanner, std::vector<std::string>* row_strings);
// Convert a kudu::Schema to a kudu::client::KuduSchema.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/1204c0a9/src/kudu/client/client-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/client/client-test.cc b/src/kudu/client/client-test.cc
index 28662dc..bc7edd5 100644
--- a/src/kudu/client/client-test.cc
+++ b/src/kudu/client/client-test.cc
@@ -1033,7 +1033,7 @@ TEST_F(ClientTest, TestGetTabletServerBlacklist) {
if (tservers.size() == 3) {
break;
}
- rt->InvalidateCachedReplicas();
+ rt->MarkStale();
SleepFor(MonoDelta::FromMilliseconds(10));
}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/1204c0a9/src/kudu/client/meta_cache.cc
----------------------------------------------------------------------
diff --git a/src/kudu/client/meta_cache.cc b/src/kudu/client/meta_cache.cc
index 8d7e427..417be2e 100644
--- a/src/kudu/client/meta_cache.cc
+++ b/src/kudu/client/meta_cache.cc
@@ -169,11 +169,17 @@ void RemoteTablet::Refresh(const TabletServerMap& tservers,
rep.failed = false;
replicas_.push_back(rep);
}
+ stale_ = false;
}
-void RemoteTablet::InvalidateCachedReplicas() {
+void RemoteTablet::MarkStale() {
lock_guard<simple_spinlock> l(&lock_);
- replicas_.clear();
+ stale_ = true;
+}
+
+bool RemoteTablet::stale() const {
+ lock_guard<simple_spinlock> l(&lock_);
+ return stale_;
}
bool RemoteTablet::MarkReplicaFailed(RemoteTabletServer *ts,
@@ -584,9 +590,14 @@ bool MetaCache::LookupTabletByKeyFastPath(const KuduTable* table,
return false;
}
+ // Stale entries must be re-fetched.
+ if ((*r)->stale()) {
+ return false;
+ }
+
if ((*r)->partition().partition_key_end().compare(partition_key) > 0 ||
(*r)->partition().partition_key_end().empty()) {
- // partition_key < partition.end OR tablet doesn't end
+ // partition_key < partition.end OR tablet doesn't end.
*remote_tablet = *r;
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/1204c0a9/src/kudu/client/meta_cache.h
----------------------------------------------------------------------
diff --git a/src/kudu/client/meta_cache.h b/src/kudu/client/meta_cache.h
index 22caf84..f99df73 100644
--- a/src/kudu/client/meta_cache.h
+++ b/src/kudu/client/meta_cache.h
@@ -123,7 +123,8 @@ class RemoteTablet : public RefCountedThreadSafe<RemoteTablet> {
RemoteTablet(std::string tablet_id,
Partition partition)
: tablet_id_(std::move(tablet_id)),
- partition_(std::move(partition)) {
+ partition_(std::move(partition)),
+ stale_(false) {
}
// Updates this tablet's replica locations.
@@ -131,6 +132,15 @@ class RemoteTablet : public RefCountedThreadSafe<RemoteTablet> {
const google::protobuf::RepeatedPtrField
<master::TabletLocationsPB_ReplicaPB>& replicas);
+ // Mark this tablet as stale, indicating that the cached tablet metadata is
+ // out of date. Staleness is checked by the MetaCache when
+ // LookupTabletByKey() is called to determine whether the fast (non-network)
+ // path can be used or whether the metadata must be refreshed from the Master.
+ void MarkStale();
+
+ // Whether the tablet has been marked as stale.
+ bool stale() const;
+
// Mark any replicas of this tablet hosted by 'ts' as failed. They will
// not be returned in future cache lookups.
//
@@ -173,10 +183,6 @@ class RemoteTablet : public RefCountedThreadSafe<RemoteTablet> {
// Return stringified representation of the list of replicas for this tablet.
std::string ReplicasAsString() const;
- // Invalidate the current set of replicas. This will result in a new lookup of the
- // replicas from the master on the next access.
- void InvalidateCachedReplicas();
-
private:
// Same as ReplicasAsString(), except that the caller must hold lock_.
std::string ReplicasAsStringUnlocked() const;
@@ -186,6 +192,7 @@ class RemoteTablet : public RefCountedThreadSafe<RemoteTablet> {
// All non-const members are protected by 'lock_'.
mutable simple_spinlock lock_;
+ bool stale_;
std::vector<RemoteReplica> replicas_;
DISALLOW_COPY_AND_ASSIGN(RemoteTablet);
@@ -254,13 +261,13 @@ class MetaCache : public RefCountedThreadSafe<MetaCache> {
rw_spinlock lock_;
- // Cache of tablet servers, by UUID.
+ // Cache of Tablet Server locations: TS UUID -> RemoteTabletServer*.
//
// Given that the set of tablet servers is bounded by physical machines, we never
// evict entries from this map until the MetaCache is destructed. So, no need to use
// shared_ptr, etc.
//
- // Protected by lock_
+ // Protected by lock_.
TabletServerMap ts_cache_;
// Cache of tablets, keyed by table ID, then by start partition key.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/1204c0a9/src/kudu/client/scanner-internal.cc
----------------------------------------------------------------------
diff --git a/src/kudu/client/scanner-internal.cc b/src/kudu/client/scanner-internal.cc
index 852a4c3..3b45a0b 100644
--- a/src/kudu/client/scanner-internal.cc
+++ b/src/kudu/client/scanner-internal.cc
@@ -154,29 +154,47 @@ Status KuduScanner::Data::CanBeRetried(const bool isNewScan,
// - TABLET_NOT_RUNNING : The scan can be retried at a different tablet server, subject
// to the client's specified selection criteria.
//
+ // - TABLET_NOT_FOUND : The scan can be retried at a different tablet server, subject
+ // to the client's specified selection criteria.
+ // The metadata for this tablet should be refreshed.
+ //
// - Any other error : Fatal. This indicates an unexpected error while processing the scan
// request.
if (rpc_status.ok() && !server_status.ok()) {
const tserver::TabletServerErrorPB& error = last_response_.error();
- if (error.code() == tserver::TabletServerErrorPB::SCANNER_EXPIRED) {
- VLOG(1) << "Got SCANNER_EXPIRED error code, non-fatal error.";
- } else if (error.code() == tserver::TabletServerErrorPB::TABLET_NOT_RUNNING) {
- VLOG(1) << "Got TABLET_NOT_RUNNING error code, temporarily blacklisting node "
- << ts_->permanent_uuid();
- blacklist->insert(ts_->permanent_uuid());
- // We've blacklisted all the live candidate tservers.
- // Do a short random sleep, clear the temp blacklist, then do another round of retries.
- if (!candidates.empty() && candidates.size() == blacklist->size()) {
- MonoDelta sleep_delta = MonoDelta::FromMilliseconds((random() % 5000) + 1000);
- LOG(INFO) << "All live candidate nodes are unavailable because of transient errors."
- << " Sleeping for " << sleep_delta.ToMilliseconds() << " ms before trying again.";
- SleepFor(sleep_delta);
- blacklist->clear();
+ switch (error.code()) {
+ case tserver::TabletServerErrorPB::SCANNER_EXPIRED:
+ VLOG(1) << "Got SCANNER_EXPIRED error code, non-fatal error.";
+ break;
+ case tserver::TabletServerErrorPB::TABLET_NOT_RUNNING:
+ VLOG(1) << "Got error code " << tserver::TabletServerErrorPB::Code_Name(error.code())
+ << ": temporarily blacklisting node " << ts_->permanent_uuid();
+ blacklist->insert(ts_->permanent_uuid());
+ // We've blacklisted all the live candidate tservers.
+ // Do a short random sleep, clear the temp blacklist, then do another round of retries.
+ if (!candidates.empty() && candidates.size() == blacklist->size()) {
+ MonoDelta sleep_delta = MonoDelta::FromMilliseconds((random() % 5000) + 1000);
+ LOG(INFO) << "All live candidate nodes are unavailable because of transient errors."
+ << " Sleeping for " << sleep_delta.ToMilliseconds() << " ms before trying again.";
+ SleepFor(sleep_delta);
+ blacklist->clear();
+ }
+ break;
+ case tserver::TabletServerErrorPB::TABLET_NOT_FOUND: {
+ // There was either a tablet configuration change or the table was
+ // deleted, since at the time of this writing we don't support splits.
+ // Backoff, then force a re-fetch of the tablet metadata.
+ remote_->MarkStale();
+ // TODO: Only backoff on the second time we hit TABLET_NOT_FOUND on the
+ // same tablet (see KUDU-1314).
+ MonoDelta backoff_time = MonoDelta::FromMilliseconds((random() % 1000) + 500);
+ SleepFor(backoff_time);
+ break;
}
- } else {
- // All other server errors are fatal. Usually indicates a malformed request, e.g. a bad scan
- // specification.
- return server_status;
+ default:
+ // All other server errors are fatal. Usually indicates a malformed request, e.g. a bad scan
+ // specification.
+ return server_status;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/1204c0a9/src/kudu/integration-tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/CMakeLists.txt b/src/kudu/integration-tests/CMakeLists.txt
index 18462e5..0631da0 100644
--- a/src/kudu/integration-tests/CMakeLists.txt
+++ b/src/kudu/integration-tests/CMakeLists.txt
@@ -43,6 +43,7 @@ set(KUDU_TEST_LINK_LIBS integration-tests ${KUDU_MIN_TEST_LIBS})
ADD_KUDU_TEST(alter_table-test)
ADD_KUDU_TEST(alter_table-randomized-test)
ADD_KUDU_TEST(registration-test RESOURCE_LOCK "master-web-port")
+ADD_KUDU_TEST(client_failover-itest)
ADD_KUDU_TEST(client-stress-test
RESOURCE_LOCK "master-rpc-ports"
RUN_SERIAL true)
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/1204c0a9/src/kudu/integration-tests/client_failover-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/client_failover-itest.cc b/src/kudu/integration-tests/client_failover-itest.cc
new file mode 100644
index 0000000..c3e3d0a
--- /dev/null
+++ b/src/kudu/integration-tests/client_failover-itest.cc
@@ -0,0 +1,151 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <boost/optional.hpp>
+#include <memory>
+#include <set>
+#include <unordered_map>
+
+#include "kudu/client/client-test-util.h"
+#include "kudu/common/wire_protocol.h"
+#include "kudu/gutil/map-util.h"
+#include "kudu/integration-tests/external_mini_cluster-itest-base.h"
+#include "kudu/integration-tests/test_workload.h"
+
+using kudu::client::CountTableRows;
+using kudu::client::KuduTable;
+using kudu::client::sp::shared_ptr;
+using kudu::itest::TServerDetails;
+using kudu::tablet::TABLET_DATA_TOMBSTONED;
+using std::set;
+using std::string;
+using std::vector;
+using std::unordered_map;
+
+namespace kudu {
+
+// Integration test for client failover behavior.
+class ClientFailoverITest : public ExternalMiniClusterITestBase {
+};
+
+// Test that we can delete the leader replica while scanning it and still get
+// results back.
+TEST_F(ClientFailoverITest, TestDeleteLeaderWhileScanning) {
+ const MonoDelta kTimeout = MonoDelta::FromSeconds(30);
+
+ vector<string> ts_flags = { "--enable_leader_failure_detection=false",
+ "--enable_remote_bootstrap=false" };
+ vector<string> master_flags = { "--catalog_manager_wait_for_new_tablets_to_elect_leader=false" };
+
+ // Start up with 4 tablet servers.
+ NO_FATALS(StartCluster(ts_flags, master_flags, 4));
+
+ // Create the test table.
+ TestWorkload workload(cluster_.get());
+ workload.set_write_timeout_millis(kTimeout.ToMilliseconds());
+ workload.Setup();
+
+ // Figure out the tablet id.
+ ASSERT_OK(inspect_->WaitForReplicaCount(3));
+ vector<string> tablets = inspect_->ListTablets();
+ ASSERT_EQ(1, tablets.size());
+ const string& tablet_id = tablets[0];
+
+ // Record the locations of the tablet replicas and the one TS that doesn't have a replica.
+ int missing_replica_index = -1;
+ set<int> replica_indexes;
+ unordered_map<string, itest::TServerDetails*> active_ts_map;
+ for (int i = 0; i < cluster_->num_tablet_servers(); i++) {
+ if (inspect_->ListTabletsOnTS(i).empty()) {
+ missing_replica_index = i;
+ } else {
+ replica_indexes.insert(i);
+ TServerDetails* ts = ts_map_[cluster_->tablet_server(i)->uuid()];
+ active_ts_map[ts->uuid()] = ts;
+ ASSERT_OK(WaitUntilTabletRunning(ts_map_[cluster_->tablet_server(i)->uuid()], tablet_id,
+ kTimeout));
+ }
+ }
+ int leader_index = *replica_indexes.begin();
+ TServerDetails* leader = ts_map_[cluster_->tablet_server(leader_index)->uuid()];
+ ASSERT_OK(itest::StartElection(leader, tablet_id, kTimeout));
+
+ // Write data to a tablet.
+ workload.Start();
+ while (workload.rows_inserted() < 100) {
+ SleepFor(MonoDelta::FromMilliseconds(10));
+ }
+ workload.StopAndJoin();
+
+ // We don't want the leader that takes over after we kill the first leader to
+ // be unsure whether the writes have been committed, so wait until all
+ // replicas have all of the writes.
+ ASSERT_OK(WaitForServersToAgree(kTimeout, active_ts_map, tablet_id,
+ workload.batches_completed() + 1));
+
+ // Open the scanner and count the rows.
+ shared_ptr<KuduTable> table;
+ ASSERT_OK(client_->OpenTable(TestWorkload::kDefaultTableName, &table));
+ ASSERT_EQ(workload.rows_inserted(), CountTableRows(table.get()));
+ LOG(INFO) << "Number of rows: " << workload.rows_inserted();
+
+ // Delete the leader replica. This will cause the next scan to the same
+ // leader to get a TABLET_NOT_FOUND error.
+ ASSERT_OK(itest::DeleteTablet(leader, tablet_id, TABLET_DATA_TOMBSTONED,
+ boost::none, kTimeout));
+
+ int old_leader_index = leader_index;
+ TServerDetails* old_leader = leader;
+ leader_index = *(++replica_indexes.begin()); // Select the "next" replica as leader.
+ leader = ts_map_[cluster_->tablet_server(leader_index)->uuid()];
+
+ ASSERT_EQ(1, replica_indexes.erase(old_leader_index));
+ ASSERT_EQ(1, active_ts_map.erase(old_leader->uuid()));
+
+ // We need to elect a new leader to remove the old node.
+ ASSERT_OK(itest::StartElection(leader, tablet_id, kTimeout));
+ ASSERT_OK(WaitForServersToAgree(kTimeout, active_ts_map, tablet_id,
+ workload.batches_completed() + 2));
+
+ // Do a config change to remove the old replica and add a new one.
+ // Cause the new replica to become leader, then do the scan again.
+ ASSERT_OK(RemoveServer(leader, tablet_id, old_leader, boost::none, kTimeout));
+
+ TServerDetails* to_add = ts_map_[cluster_->tablet_server(missing_replica_index)->uuid()];
+ ASSERT_OK(AddServer(leader, tablet_id, to_add, consensus::RaftPeerPB::VOTER,
+ boost::none, kTimeout));
+ HostPort hp;
+ ASSERT_OK(HostPortFromPB(leader->registration.rpc_addresses(0), &hp));
+ ASSERT_OK(StartRemoteBootstrap(to_add, tablet_id, leader->uuid(), hp, 1, kTimeout));
+
+ const string& new_ts_uuid = cluster_->tablet_server(missing_replica_index)->uuid();
+ InsertOrDie(&replica_indexes, missing_replica_index);
+ InsertOrDie(&active_ts_map, new_ts_uuid, ts_map_[new_ts_uuid]);
+
+ // Wait for remote bootstrap to complete. Then elect the new node.
+ ASSERT_OK(WaitForServersToAgree(kTimeout, active_ts_map, tablet_id,
+ workload.batches_completed() + 4));
+ leader_index = missing_replica_index;
+ leader = ts_map_[cluster_->tablet_server(leader_index)->uuid()];
+ ASSERT_OK(itest::StartElection(leader, tablet_id, kTimeout));
+ ASSERT_OK(WaitForServersToAgree(kTimeout, active_ts_map, tablet_id,
+ workload.batches_completed() + 5));
+
+ ASSERT_EQ(workload.rows_inserted(), CountTableRows(table.get()));
+}
+
+} // namespace kudu
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/1204c0a9/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc b/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc
index 8c2158d..41c4f24 100644
--- a/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc
+++ b/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc
@@ -18,6 +18,7 @@
#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h"
#include <algorithm>
+#include <set>
#include "kudu/consensus/metadata.pb.h"
#include "kudu/gutil/strings/join.h"
@@ -32,6 +33,7 @@
namespace kudu {
namespace itest {
+using std::set;
using std::string;
using std::vector;
@@ -81,6 +83,14 @@ int ExternalMiniClusterFsInspector::CountWALSegmentsOnTS(int index) {
return total_segments;
}
+vector<string> ExternalMiniClusterFsInspector::ListTablets() {
+ set<string> tablets;
+ for (int i = 0; i < cluster_->num_tablet_servers(); i++) {
+ auto ts_tablets = ListTabletsOnTS(i);
+ tablets.insert(ts_tablets.begin(), ts_tablets.end());
+ }
+ return vector<string>(tablets.begin(), tablets.end());
+}
vector<string> ExternalMiniClusterFsInspector::ListTabletsOnTS(int index) {
string data_dir = cluster_->tablet_server(index)->data_dir();
string meta_dir = JoinPathSegments(data_dir, FsManager::kTabletMetadataDirName);
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/1204c0a9/src/kudu/integration-tests/external_mini_cluster_fs_inspector.h
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/external_mini_cluster_fs_inspector.h b/src/kudu/integration-tests/external_mini_cluster_fs_inspector.h
index 2bf3023..ebe038b 100644
--- a/src/kudu/integration-tests/external_mini_cluster_fs_inspector.h
+++ b/src/kudu/integration-tests/external_mini_cluster_fs_inspector.h
@@ -52,6 +52,7 @@ class ExternalMiniClusterFsInspector {
Status ListFilesInDir(const std::string& path, std::vector<std::string>* entries);
int CountFilesInDir(const std::string& path);
int CountWALSegmentsOnTS(int index);
+ std::vector<std::string> ListTablets();
std::vector<std::string> ListTabletsOnTS(int index);
int CountWALSegmentsForTabletOnTS(int index, const std::string& tablet_id);
bool DoesConsensusMetaExistForTabletOnTS(int index, const std::string& tablet_id);