You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by mp...@apache.org on 2018/05/29 23:44:26 UTC
[1/5] kudu git commit: [tools] ksck imp. [8/n]: Easy-to-see errors
when running w/o admin
Repository: kudu
Updated Branches:
refs/heads/master 2f61b72ca -> e840ad7e9
[tools] ksck imp. [8/n]: Easy-to-see errors when running w/o admin
If ksck is not run as an admin user, then it cannot gather all of the
information it uses to do cluster checks. "Not authorized" errors are
easy to mistake for other errors. This patch handles these errors
specially to make it obvious to administrators when they have forgotten
to run as an administrative user.
The new errors at the end of the ksck output look like this:
==================
Errors:
==================
Not authorized: error fetching info from masters: failed to gather info from 1 of 1 masters due to lack of admin privileges
Not authorized: error fetching info from tablet servers: failed to gather info from 11 of 12 tablet servers due to lack of admin privileges
FAILED
Not authorized: re-run ksck with administrator privileges
Change-Id: I644957103efefceeba4b82f4557376a603507423
Reviewed-on: http://gerrit.cloudera.org:8080/10300
Reviewed-by: Attila Bukor <ab...@cloudera.com>
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin <as...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/e04fd4e4
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/e04fd4e4
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/e04fd4e4
Branch: refs/heads/master
Commit: e04fd4e4d935a062235d6f7451db0ce56773206a
Parents: 2f61b72
Author: Will Berkeley <wd...@apache.org>
Authored: Thu May 24 16:10:39 2018 -0700
Committer: Will Berkeley <wd...@gmail.com>
Committed: Fri May 25 21:03:59 2018 +0000
----------------------------------------------------------------------
src/kudu/tools/ksck-test.cc | 22 +++++++++++++++++++
src/kudu/tools/ksck.cc | 43 ++++++++++++++++++++++++++++++++++++-
src/kudu/tools/ksck_results.cc | 11 ++++++++--
src/kudu/tools/ksck_results.h | 3 +++
src/kudu/tools/tool.proto | 1 +
5 files changed, 77 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/e04fd4e4/src/kudu/tools/ksck-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/ksck-test.cc b/src/kudu/tools/ksck-test.cc
index eff2609..64bca59 100644
--- a/src/kudu/tools/ksck-test.cc
+++ b/src/kudu/tools/ksck-test.cc
@@ -756,6 +756,28 @@ TEST_F(KsckTest, TestMasterUnavailable) {
CheckJsonStringVsKsckResults(KsckResultsToJsonString(), ksck_->results());
}
+TEST_F(KsckTest, TestUnauthorized) {
+ Status noauth = Status::RemoteError("Not authorized: unauthorized access to method");
+ shared_ptr<MockKsckMaster> master =
+ std::static_pointer_cast<MockKsckMaster>(cluster_->masters_.at(1));
+ master->fetch_info_status_ = noauth;
+ shared_ptr<MockKsckTabletServer> tserver =
+ std::static_pointer_cast<MockKsckTabletServer>(
+ cluster_->tablet_servers().begin()->second);
+ tserver->fetch_info_status_ = noauth;
+ Status s = RunKsck();
+ ASSERT_TRUE(s.IsNotAuthorized()) << s.ToString();
+ ASSERT_STR_CONTAINS(s.ToString(), "re-run ksck with administrator privileges");
+ ASSERT_STR_CONTAINS(err_stream_.str(),
+ "failed to gather info from 1 of 3 "
+ "masters due to lack of admin privileges");
+ ASSERT_STR_CONTAINS(err_stream_.str(),
+ "failed to gather info from 1 of 3 "
+ "tablet servers due to lack of admin privileges");
+
+ CheckJsonStringVsKsckResults(KsckResultsToJsonString(), ksck_->results());
+}
+
// A wrong-master-uuid situation can happen if a master that is part of, e.g.,
// a 3-peer config fails permanently and is wiped and reborn on the same address
// with a new uuid.
http://git-wip-us.apache.org/repos/asf/kudu/blob/e04fd4e4/src/kudu/tools/ksck.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/ksck.cc b/src/kudu/tools/ksck.cc
index 5e36f79..6b886d0 100644
--- a/src/kudu/tools/ksck.cc
+++ b/src/kudu/tools/ksck.cc
@@ -142,6 +142,11 @@ void BuildKsckConsensusStateForConfigMember(const consensus::ConsensusStatePB& c
}
}
+bool IsNotAuthorizedMethodAccess(const Status& s) {
+ return s.IsRemoteError() &&
+ s.ToString().find("Not authorized: unauthorized access to method") != string::npos;
+}
+
} // anonymous namespace
ChecksumOptions::ChecksumOptions()
@@ -192,6 +197,7 @@ Ksck::Ksck(shared_ptr<KsckCluster> cluster, ostream* out)
Status Ksck::CheckMasterHealth() {
int bad_masters = 0;
+ int unauthorized_masters = 0;
vector<KsckServerHealthSummary> master_summaries;
// There shouldn't be more than 5 masters, so we'll keep it simple and gather
// info in sequence instead of spreading it across a threadpool.
@@ -204,14 +210,28 @@ Status Ksck::CheckMasterHealth() {
sh.address = master->address();
sh.status = s;
if (!s.ok()) {
+ if (IsNotAuthorizedMethodAccess(s)) {
+ sh.health = KsckServerHealth::UNAUTHORIZED;
+ unauthorized_masters++;
+ } else {
+ sh.health = KsckServerHealth::UNAVAILABLE;
+ }
bad_masters++;
- sh.health = KsckServerHealth::UNAVAILABLE;
}
master_summaries.push_back(std::move(sh));
}
results_.master_summaries.swap(master_summaries);
int num_masters = cluster_->masters().size();
+
+ // Return a NotAuthorized status if any master has auth errors, since this
+ // indicates ksck may not be able to gather full and accurate info.
+ if (unauthorized_masters > 0) {
+ return Status::NotAuthorized(
+ Substitute("failed to gather info from $0 of $1 "
+ "masters due to lack of admin privileges",
+ unauthorized_masters, num_masters));
+ }
if (bad_masters > 0) {
return Status::NetworkError(
Substitute("failed to gather info from all masters: $0 of $1 had errors",
@@ -283,6 +303,7 @@ Status Ksck::FetchInfoFromTabletServers() {
.Build(&pool));
AtomicInt<int32_t> bad_servers(0);
+ AtomicInt<int32_t> unauthorized_servers(0);
VLOG(1) << "Fetching info from all " << servers_count << " tablet servers";
vector<KsckServerHealthSummary> tablet_server_summaries;
@@ -304,6 +325,10 @@ Status Ksck::FetchInfoFromTabletServers() {
summary.address = entry.second->address();
summary.status = s;
if (!s.ok()) {
+ if (IsNotAuthorizedMethodAccess(s)) {
+ health = KsckServerHealth::UNAUTHORIZED;
+ unauthorized_servers.Increment();
+ }
bad_servers.Increment();
}
summary.health = health;
@@ -319,6 +344,14 @@ Status Ksck::FetchInfoFromTabletServers() {
if (bad_servers.Load() == 0) {
return Status::OK();
}
+ // Return a NotAuthorized status if any tablet server has auth errors, since
+ // this indicates ksck may not be able to gather full and accurate info.
+ if (unauthorized_servers.Load() > 0) {
+ return Status::NotAuthorized(
+ Substitute("failed to gather info from $0 of $1 tablet servers due "
+ "to lack of admin privileges",
+ unauthorized_servers.Load(), servers_count));
+ }
return Status::NetworkError(
Substitute("failed to gather info for all tablet servers: $0 of $1 had errors",
bad_servers.Load(), servers_count));
@@ -357,6 +390,14 @@ Status Ksck::Run() {
results_.error_messages, "checksum scan error");
}
+ // Use a special-case error if there are auth errors. This makes it harder
+ // for admins to miss that ksck isn't working right because they forgot to
+ // (e.g.) sudo -u kudu when running ksck!
+ if (std::any_of(std::begin(results_.error_messages),
+ std::end(results_.error_messages),
+ [](const Status& s) { return s.IsNotAuthorized(); })) {
+ return Status::NotAuthorized("re-run ksck with administrator privileges");
+ }
if (!results_.error_messages.empty()) {
return Status::RuntimeError("ksck discovered errors");
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/e04fd4e4/src/kudu/tools/ksck_results.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/ksck_results.cc b/src/kudu/tools/ksck_results.cc
index a3b01af..476db0e 100644
--- a/src/kudu/tools/ksck_results.cc
+++ b/src/kudu/tools/ksck_results.cc
@@ -153,6 +153,8 @@ const char* const ServerHealthToString(KsckServerHealth sh) {
switch (sh) {
case KsckServerHealth::HEALTHY:
return "HEALTHY";
+ case KsckServerHealth::UNAUTHORIZED:
+ return "UNAUTHORIZED";
case KsckServerHealth::UNAVAILABLE:
return "UNAVAILABLE";
case KsckServerHealth::WRONG_SERVER_UUID:
@@ -166,10 +168,12 @@ int ServerHealthScore(KsckServerHealth sh) {
switch (sh) {
case KsckServerHealth::HEALTHY:
return 0;
- case KsckServerHealth::UNAVAILABLE:
+ case KsckServerHealth::UNAUTHORIZED:
return 1;
- case KsckServerHealth::WRONG_SERVER_UUID:
+ case KsckServerHealth::UNAVAILABLE:
return 2;
+ case KsckServerHealth::WRONG_SERVER_UUID:
+ return 3;
default:
LOG(FATAL) << "Unknown KsckServerHealth";
}
@@ -447,6 +451,9 @@ void KsckServerHealthSummaryToPb(const KsckServerHealthSummary& summary,
case KsckServerHealth::HEALTHY:
pb->set_health(KsckServerHealthSummaryPB_ServerHealth_HEALTHY);
break;
+ case KsckServerHealth::UNAUTHORIZED:
+ pb->set_health(KsckServerHealthSummaryPB_ServerHealth_UNAUTHORIZED);
+ break;
case KsckServerHealth::UNAVAILABLE:
pb->set_health(KsckServerHealthSummaryPB_ServerHealth_UNAVAILABLE);
break;
http://git-wip-us.apache.org/repos/asf/kudu/blob/e04fd4e4/src/kudu/tools/ksck_results.h
----------------------------------------------------------------------
diff --git a/src/kudu/tools/ksck_results.h b/src/kudu/tools/ksck_results.h
index c9fba92..a7b9c02 100644
--- a/src/kudu/tools/ksck_results.h
+++ b/src/kudu/tools/ksck_results.h
@@ -119,6 +119,9 @@ enum class KsckServerHealth {
// The server is healthy.
HEALTHY,
+ // The server rejected attempts to communicate as unauthorized.
+ UNAUTHORIZED,
+
// The server can't be contacted.
UNAVAILABLE,
http://git-wip-us.apache.org/repos/asf/kudu/blob/e04fd4e4/src/kudu/tools/tool.proto
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool.proto b/src/kudu/tools/tool.proto
index 63e3961..f293549 100644
--- a/src/kudu/tools/tool.proto
+++ b/src/kudu/tools/tool.proto
@@ -221,6 +221,7 @@ message KsckServerHealthSummaryPB {
HEALTHY = 0;
UNAVAILABLE = 1;
WRONG_SERVER_UUID = 2;
+ UNAUTHORIZED = 3;
}
optional string uuid = 1;
optional string address = 2;
[3/5] kudu git commit: [tools] ksck improvements [5/n]: Checks for
experimental, unsafe, hidden flags
Posted by mp...@apache.org.
[tools] ksck improvements [5/n]: Checks for experimental, unsafe, hidden flags
This adds checks to ksck that look for experimental, unsafe, and hidden
flags set to non-default values on Kudu masters and tablet servers. If
any are found, ksck generates a table summarizing the different flags and
their values. For example:
Flag | Value | Tags | Master
------------------------+---------------------+----------------------+-------------------------------
codegen_dump_functions | true | runtime,experimental | localhost:7052,localhost:7053
min_compression_ratio | 0.80000000000000004 | experimental | all 3 server(s) checked
safe_time_max_lag_ms | 40000 | experimental | localhost:7052
safe_time_max_lag_ms | 50000 | experimental | localhost:7053
The table has one row for each unique (flag, value) pair, listing all
daemons with --flag=value. So, in the above output, there are two rows
for the flag --safe_time_max_lag_ms because it's set to two different
values on two masters. This makes it easy to scan for concerning flags
and their values.
Since the output might not scale to a large number of
servers, the CSV of servers is abbreviated, by default, to 3 entries,
with the number of additional servers indicated. The number of entries
before truncation kicks in is controlled by --truncate_server_csv_length.
Additionally, if all checked servers have an unusual --flag=value we call
that out specially. For example, the above table reprinted with
--truncate_server_csv_length=1 would look like
Flag | Value | Tags | Master
------------------------+---------------------+----------------------+--------------------------------------
codegen_dump_functions | true | runtime,experimental | localhost:7052 and 1 other server(s)
min_compression_ratio | 0.80000000000000004 | experimental | all 3 server(s) checked
safe_time_max_lag_ms | 40000 | experimental | localhost:7052
safe_time_max_lag_ms | 50000 | experimental | localhost:7053
assuming that there are 3 servers checked in total.
Having unusual flags or failing to gather flags isn't considered an
error, since it doesn't indicate the cluster is unhealthy (in the latter
case because the daemon may not support the GetFlags RPC). Instead,
flag checks surface their results in a new warnings section near the
end of the ksck output.
The new warnings section looks like this in context:
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
unable to get flag information for tablet server 812db6461bae4f62a651e132f783ab53 (127.0.0.1:7250): could not get status from server: Client connection negotiation failed: client connection to 127.0.0.1:7250: connect: Connection refused (error 61)
Some tablet servers have unsafe, experimental, or hidden flags set
tserver flag check error: 1 of 3 tservers' flags were not available
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
FAILED
Runtime error: ksck discovered errors
Change-Id: Idd6c179e5256b2f2bae2f7486c5e0365ef184706
Reviewed-on: http://gerrit.cloudera.org:8080/10061
Tested-by: Kudu Jenkins
Reviewed-by: Andrew Wong <aw...@cloudera.com>
Reviewed-by: Attila Bukor <ab...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/66e41bda
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/66e41bda
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/66e41bda
Branch: refs/heads/master
Commit: 66e41bda1d628f11b4ae121fd2da63c78ba89c83
Parents: 98ef3c0
Author: Will Berkeley <wd...@apache.org>
Authored: Thu Apr 12 15:24:52 2018 -0700
Committer: Will Berkeley <wd...@gmail.com>
Committed: Tue May 29 17:11:50 2018 +0000
----------------------------------------------------------------------
src/kudu/tools/ksck-test.cc | 112 ++++++++++++++++++++++++++++++++
src/kudu/tools/ksck.cc | 97 ++++++++++++++++++++++++++-
src/kudu/tools/ksck.h | 68 ++++++++++++++++++-
src/kudu/tools/ksck_remote-test.cc | 36 ++++++++++
src/kudu/tools/ksck_remote.cc | 36 ++++++++++
src/kudu/tools/ksck_remote.h | 4 ++
src/kudu/tools/ksck_results.cc | 80 +++++++++++++++++++++++
src/kudu/tools/ksck_results.h | 30 +++++++++
8 files changed, 459 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/66e41bda/src/kudu/tools/ksck-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/ksck-test.cc b/src/kudu/tools/ksck-test.cc
index 64bca59..8a2e137 100644
--- a/src/kudu/tools/ksck-test.cc
+++ b/src/kudu/tools/ksck-test.cc
@@ -40,6 +40,7 @@
#include "kudu/gutil/map-util.h"
#include "kudu/gutil/port.h"
#include "kudu/gutil/strings/substitute.h"
+#include "kudu/server/server_base.pb.h"
#include "kudu/tablet/metadata.pb.h"
#include "kudu/tablet/tablet.pb.h"
#include "kudu/tools/ksck_results.h"
@@ -51,10 +52,12 @@
DECLARE_bool(checksum_scan);
DECLARE_string(color);
+DECLARE_uint32(truncate_server_csv_length);
namespace kudu {
namespace tools {
+using server::GetFlagsResponsePB;
using std::ostringstream;
using std::shared_ptr;
using std::static_pointer_cast;
@@ -70,6 +73,7 @@ class MockKsckMaster : public KsckMaster {
: KsckMaster(address),
fetch_info_status_(Status::OK()) {
uuid_ = uuid;
+ flags_ = GetFlagsResponsePB{};
}
Status Init() override {
@@ -89,11 +93,17 @@ class MockKsckMaster : public KsckMaster {
return fetch_cstate_status_;
}
+ Status FetchUnusualFlags() override {
+ flags_state_ = KsckFetchState::FETCHED;
+ return Status::OK();
+ }
+
// Public because the unit tests mutate these variables directly.
Status fetch_info_status_;
Status fetch_cstate_status_;
using KsckMaster::uuid_;
using KsckMaster::cstate_;
+ using KsckMaster::flags_;
};
class MockKsckTabletServer : public KsckTabletServer {
@@ -103,6 +113,7 @@ class MockKsckTabletServer : public KsckTabletServer {
fetch_info_status_(Status::OK()),
fetch_info_health_(KsckServerHealth::HEALTHY),
address_("<mock>") {
+ flags_ = GetFlagsResponsePB{};
}
Status FetchInfo(KsckServerHealth* health) override {
@@ -121,6 +132,11 @@ class MockKsckTabletServer : public KsckTabletServer {
return Status::OK();
}
+ Status FetchUnusualFlags() override {
+ flags_state_ = KsckFetchState::FETCHED;
+ return Status::OK();
+ }
+
virtual void RunTabletChecksumScanAsync(
const std::string& /*tablet_id*/,
const Schema& /*schema*/,
@@ -137,6 +153,7 @@ class MockKsckTabletServer : public KsckTabletServer {
// Public because the unit tests mutate these variables directly.
Status fetch_info_status_;
KsckServerHealth fetch_info_health_;
+ using KsckTabletServer::flags_;
private:
const string address_;
@@ -847,6 +864,51 @@ TEST_F(KsckTest, TestLeaderMasterUnavailable) {
CheckJsonStringVsKsckResults(KsckResultsToJsonString(), ksck_->results());
}
+TEST_F(KsckTest, TestMasterFlagCheck) {
+ // Setup flags for each mock master.
+ for (int i = 0; i < cluster_->masters().size(); i++) {
+ server::GetFlagsResponsePB flags;
+ {
+ // Add an experimental flag with the same value for each master.
+ auto* flag = flags.add_flags();
+ flag->set_name("experimental_flag");
+ flag->set_value("x");
+ flag->mutable_tags()->Add("experimental");
+ }
+ {
+ // Add a hidden flag with a different value for each master.
+ auto* flag = flags.add_flags();
+ flag->set_name("hidden_flag");
+ flag->set_value(std::to_string(i));
+ flag->mutable_tags()->Add("hidden");
+ }
+ {
+ // Add a hidden and unsafe flag with one master having a different value
+ // than the other two.
+ auto* flag = flags.add_flags();
+ flag->set_name("hidden_unsafe_flag");
+ flag->set_value(std::to_string(i % 2));
+ flag->mutable_tags()->Add("hidden");
+ flag->mutable_tags()->Add("unsafe");
+ }
+ shared_ptr<MockKsckMaster> master =
+ std::static_pointer_cast<MockKsckMaster>(cluster_->masters_.at(i));
+ master->flags_ = flags;
+ }
+ ASSERT_OK(ksck_->CheckMasterHealth());
+ ASSERT_OK(ksck_->CheckMasterUnusualFlags());
+ ASSERT_OK(ksck_->PrintResults());
+ ASSERT_STR_CONTAINS(err_stream_.str(),
+ " Flag | Value | Tags | Master\n"
+ "--------------------+-------+---------------+-------------------------\n"
+ " experimental_flag | x | experimental | all 3 server(s) checked\n"
+ " hidden_flag | 0 | hidden | master-0\n"
+ " hidden_flag | 1 | hidden | master-1\n"
+ " hidden_flag | 2 | hidden | master-2\n"
+ " hidden_unsafe_flag | 0 | hidden,unsafe | master-0, master-2\n"
+ " hidden_unsafe_flag | 1 | hidden,unsafe | master-1");
+}
+
TEST_F(KsckTest, TestWrongUUIDTabletServer) {
CreateOneTableOneTablet();
@@ -923,6 +985,56 @@ TEST_F(KsckTest, TestBadTabletServer) {
CheckJsonStringVsKsckResults(KsckResultsToJsonString(), ksck_->results());
}
+TEST_F(KsckTest, TestTserverFlagCheck) {
+ // Lower the truncation threshold to test truncation.
+ FLAGS_truncate_server_csv_length = 1;
+
+ // Setup flags for each mock tablet server.
+ int i = 0;
+ for (const auto& entry : cluster_->tablet_servers()) {
+ server::GetFlagsResponsePB flags;
+ {
+ // Add an experimental flag with the same value for each tablet server.
+ auto* flag = flags.add_flags();
+ flag->set_name("experimental_flag");
+ flag->set_value("x");
+ flag->mutable_tags()->Add("experimental");
+ }
+ {
+ // Add a hidden flag with a different value for each tablet server.
+ auto* flag = flags.add_flags();
+ flag->set_name("hidden_flag");
+ flag->set_value(std::to_string(i));
+ flag->mutable_tags()->Add("hidden");
+ }
+ {
+ // Add a hidden and unsafe flag with one tablet server having a different value
+ // than the other two.
+ auto* flag = flags.add_flags();
+ flag->set_name("hidden_unsafe_flag");
+ flag->set_value(std::to_string(i % 2));
+ flag->mutable_tags()->Add("hidden");
+ flag->mutable_tags()->Add("unsafe");
+ }
+ shared_ptr<MockKsckTabletServer> ts =
+ std::static_pointer_cast<MockKsckTabletServer>(entry.second);
+ ts->flags_ = flags;
+ i++;
+ }
+ ASSERT_OK(ksck_->FetchInfoFromTabletServers());
+ ASSERT_OK(ksck_->CheckTabletServerUnusualFlags());
+ ASSERT_OK(ksck_->PrintResults());
+ ASSERT_STR_CONTAINS(err_stream_.str(),
+ " Flag | Value | Tags | Tablet Server\n"
+ "--------------------+-------+---------------+-------------------------------\n"
+ " experimental_flag | x | experimental | all 3 server(s) checked\n"
+ " hidden_flag | 0 | hidden | <mock>\n"
+ " hidden_flag | 1 | hidden | <mock>\n"
+ " hidden_flag | 2 | hidden | <mock>\n"
+ " hidden_unsafe_flag | 0 | hidden,unsafe | <mock>, and 1 other server(s)\n"
+ " hidden_unsafe_flag | 1 | hidden,unsafe | <mock>");
+}
+
TEST_F(KsckTest, TestOneTableCheck) {
CreateOneTableOneTablet();
FLAGS_checksum_scan = true;
http://git-wip-us.apache.org/repos/asf/kudu/blob/66e41bda/src/kudu/tools/ksck.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/ksck.cc b/src/kudu/tools/ksck.cc
index 6b886d0..8357999 100644
--- a/src/kudu/tools/ksck.cc
+++ b/src/kudu/tools/ksck.cc
@@ -219,6 +219,15 @@ Status Ksck::CheckMasterHealth() {
bad_masters++;
}
master_summaries.push_back(std::move(sh));
+
+ // Fetch the flags information.
+ // Failing to gather flags is only a warning.
+ if (!master->FetchUnusualFlags().ok()) {
+ results_.warning_messages.push_back(s.CloneAndPrepend(Substitute(
+ "unable to get flag information for master $0 ($1)",
+ master->uuid(),
+ master->address())));
+ }
}
results_.master_summaries.swap(master_summaries);
@@ -279,6 +288,48 @@ Status Ksck::CheckMasterConsensus() {
return Status::OK();
}
+void Ksck::AddFlagsToFlagMaps(const server::GetFlagsResponsePB& flags,
+ const string& server_address,
+ KsckFlagToServersMap* flags_to_servers_map,
+ KsckFlagTagsMap* flag_tags_map) {
+ CHECK(flags_to_servers_map);
+ CHECK(flag_tags_map);
+ for (const auto& f : flags.flags()) {
+ const std::pair<string, string> key(f.name(), f.value());
+ if (!InsertIfNotPresent(flags_to_servers_map, key, { server_address })) {
+ FindOrDieNoPrint(*flags_to_servers_map, key).push_back(server_address);
+ }
+ InsertIfNotPresent(flag_tags_map, f.name(), JoinStrings(f.tags(), ","));
+ }
+}
+
+Status Ksck::CheckMasterUnusualFlags() {
+ int bad_masters = 0;
+ Status last_error = Status::OK();
+ for (const auto& master : cluster_->masters()) {
+ if (!master->flags()) {
+ bad_masters++;
+ continue;
+ }
+ AddFlagsToFlagMaps(*master->flags(),
+ master->address(),
+ &results_.master_flag_to_servers_map,
+ &results_.master_flag_tags_map);
+ }
+
+ if (!results_.master_flag_to_servers_map.empty()) {
+ results_.warning_messages.push_back(Status::ConfigurationError(
+ "Some masters have unsafe, experimental, or hidden flags set"));
+ }
+
+ if (bad_masters > 0) {
+ return Status::Incomplete(
+ Substitute("$0 of $1 masters' flags were not available",
+ bad_masters, cluster_->masters().size()));
+ }
+ return Status::OK();
+}
+
Status Ksck::CheckClusterRunning() {
VLOG(1) << "Connecting to the leader master";
return cluster_->Connect();
@@ -333,8 +384,19 @@ Status Ksck::FetchInfoFromTabletServers() {
}
summary.health = health;
- std::lock_guard<simple_spinlock> lock(tablet_server_summaries_lock);
- tablet_server_summaries.push_back(std::move(summary));
+ {
+ std::lock_guard<simple_spinlock> lock(tablet_server_summaries_lock);
+ tablet_server_summaries.push_back(std::move(summary));
+ }
+
+ // Fetch the flags information.
+ // Failing to gather flags is only a warning.
+ if (!ts->FetchUnusualFlags().ok()) {
+ results_.warning_messages.push_back(s.CloneAndPrepend(Substitute(
+ "unable to get flag information for tablet server $0 ($1)",
+ ts->uuid(),
+ ts->address())));
+ }
}));
}
pool->Wait();
@@ -366,6 +428,8 @@ Status Ksck::Run() {
"error fetching info from masters");
PUSH_PREPEND_NOT_OK(CheckMasterConsensus(), results_.error_messages,
"master consensus error");
+ PUSH_PREPEND_NOT_OK(CheckMasterUnusualFlags(), results_.warning_messages,
+ "master flag check error");
// CheckClusterRunning and FetchTableAndTabletInfo must succeed for
// subsequent checks to be runnable.
@@ -381,6 +445,8 @@ Status Ksck::Run() {
PUSH_PREPEND_NOT_OK(FetchInfoFromTabletServers(), results_.error_messages,
"error fetching info from tablet servers");
+ PUSH_PREPEND_NOT_OK(CheckTabletServerUnusualFlags(), results_.warning_messages,
+ "tserver flag check error");
PUSH_PREPEND_NOT_OK(CheckTablesConsistency(), results_.error_messages,
"table consistency check error");
@@ -404,6 +470,33 @@ Status Ksck::Run() {
return Status::OK();
}
+Status Ksck::CheckTabletServerUnusualFlags() {
+ int bad_tservers = 0;
+ for (const auto& uuid_and_ts : cluster_->tablet_servers()) {
+ const auto& tserver = uuid_and_ts.second;
+ if (!tserver->flags()) {
+ bad_tservers++;
+ continue;
+ }
+ AddFlagsToFlagMaps(*tserver->flags(),
+ tserver->address(),
+ &results_.tserver_flag_to_servers_map,
+ &results_.tserver_flag_tags_map);
+ }
+
+ if (!results_.tserver_flag_to_servers_map.empty()) {
+ results_.warning_messages.push_back(Status::ConfigurationError(
+ "Some tablet servers have unsafe, experimental, or hidden flags set"));
+ }
+
+ if (bad_tservers > 0) {
+ return Status::Incomplete(
+ Substitute("$0 of $1 tservers' flags were not available",
+ bad_tservers, cluster_->tablet_servers().size()));
+ }
+ return Status::OK();
+}
+
Status Ksck::PrintResults() {
PrintMode mode;
if (FLAGS_ksck_format == "plain_concise") {
http://git-wip-us.apache.org/repos/asf/kudu/blob/66e41bda/src/kudu/tools/ksck.h
----------------------------------------------------------------------
diff --git a/src/kudu/tools/ksck.h b/src/kudu/tools/ksck.h
index c2bed92..5599ee4 100644
--- a/src/kudu/tools/ksck.h
+++ b/src/kudu/tools/ksck.h
@@ -37,6 +37,7 @@
#include "kudu/consensus/metadata.pb.h"
#include "kudu/gutil/macros.h"
#include "kudu/gutil/strings/substitute.h"
+#include "kudu/server/server_base.pb.h"
#include "kudu/tablet/metadata.pb.h"
#include "kudu/tablet/tablet.pb.h" // IWYU pragma: keep
#include "kudu/tools/ksck_results.h"
@@ -221,14 +222,20 @@ class KsckMaster {
virtual Status Init() = 0;
+ // Connects to the master, checking if it is healthy and gathering basic info.
virtual Status FetchInfo() = 0;
+ // Connects to the master and populates the consensus map.
virtual Status FetchConsensusState() = 0;
+ // Retrieves "unusual" flags from the KsckMaster.
+ // "Unusual" flags ares ones tagged hidden, experimental, or unsafe.
+ virtual Status FetchUnusualFlags() = 0;
+
// Since masters are provided by address, FetchInfo() must be called before
// calling this method.
virtual const std::string& uuid() const {
- CHECK_NE(state_, KsckFetchState::UNINITIALIZED);
+ CHECK_NE(KsckFetchState::UNINITIALIZED, state_);
return uuid_;
}
@@ -237,10 +244,15 @@ class KsckMaster {
}
virtual const boost::optional<consensus::ConsensusStatePB> cstate() const {
- CHECK_NE(state_, KsckFetchState::UNINITIALIZED);
+ CHECK_NE(KsckFetchState::UNINITIALIZED, state_);
return cstate_;
}
+ virtual const boost::optional<server::GetFlagsResponsePB>& flags() const {
+ CHECK_NE(KsckFetchState::UNINITIALIZED, flags_state_);
+ return flags_;
+ }
+
std::string ToString() const {
return strings::Substitute("$0 ($1)", uuid(), address());
}
@@ -259,11 +271,21 @@ class KsckMaster {
const std::string address_;
std::string uuid_;
+
+ // state_ reflects whether the fetch of the core ksck info has been done, and
+ // if it succeeded or failed.
KsckFetchState state_ = KsckFetchState::UNINITIALIZED;
+ // flags_state_ reflects whether the fetch of the non-critical flags info has
+ // been done, and if it succeeded or failed.
+ KsckFetchState flags_state_ = KsckFetchState::UNINITIALIZED;
+
// May be none if consensus state fetch fails.
boost::optional<consensus::ConsensusStatePB> cstate_;
+ // May be none if flag fetch fails.
+ boost::optional<server::GetFlagsResponsePB> flags_;
+
private:
DISALLOW_COPY_AND_ASSIGN(KsckMaster);
};
@@ -296,6 +318,10 @@ class KsckTabletServer {
// Otherwise 'health' will be UNAVAILABLE
virtual Status FetchConsensusState(KsckServerHealth* health) = 0;
+ // Retrieves "unusual" flags from the KsckTabletServer.
+ // "Unusual" flags ares ones tagged hidden, experimental, or unsafe.
+ virtual Status FetchUnusualFlags() = 0;
+
// Executes a checksum scan on the associated tablet, and runs the callback
// with the result. The callback must be threadsafe and non-blocking.
virtual void RunTabletChecksumScanAsync(
@@ -333,6 +359,11 @@ class KsckTabletServer {
tablet::TabletStatePB ReplicaState(const std::string& tablet_id) const;
+ virtual const boost::optional<server::GetFlagsResponsePB>& flags() const {
+ CHECK_NE(KsckFetchState::UNINITIALIZED, flags_state_);
+ return flags_;
+ }
+
uint64_t current_timestamp() const {
CHECK_EQ(KsckFetchState::FETCHED, state_);
return timestamp_;
@@ -347,9 +378,19 @@ class KsckTabletServer {
FRIEND_TEST(KsckTest, TestMismatchedAssignments);
FRIEND_TEST(KsckTest, TestTabletCopying);
+ // state_ reflects whether the fetch of the core info has been done, and if
+ // it succeeded or failed.
KsckFetchState state_ = KsckFetchState::UNINITIALIZED;
+
+ // flags_state_ reflects whether the fetch of the non-critical flags info has
+ // been done, and if it succeeded or failed.
+ KsckFetchState flags_state_ = KsckFetchState::UNINITIALIZED;
+
TabletStatusMap tablet_status_map_;
TabletConsensusStateMap tablet_consensus_state_map_;
+
+ // May be none if flag fetch fails.
+ boost::optional<server::GetFlagsResponsePB> flags_;
uint64_t timestamp_;
const std::string uuid_;
@@ -455,6 +496,12 @@ class Ksck {
// Check that the masters' consensus information is consistent.
Status CheckMasterConsensus();
+ // Check for "unusual" flags on masters.
+ // "Unusual" flags are ones tagged hidden, experimental, or unsafe and set
+ // to a non-default value.
+ // Must first call CheckMasterHealth().
+ Status CheckMasterUnusualFlags();
+
// Verifies that it can connect to the cluster, i.e. that it can contact a
// leader master.
Status CheckClusterRunning();
@@ -465,8 +512,15 @@ class Ksck {
// Connects to all tablet servers, checks that they are alive, and fetches
// their current status and tablet information.
+ // Must first call FetchTableAndTabletInfo().
Status FetchInfoFromTabletServers();
+ // Check for "unusual" flags on tablet servers.
+ // "Unusual" flags are ones tagged hidden, experimental, or unsafe and set
+ // to a non-default value.
+ // Must first call FetchInfoFromTabletServers().
+ Status CheckTabletServerUnusualFlags();
+
// Verifies that all the tablets in all tables matching the filters have
// enough replicas, and that each tablet's view of the tablet's consensus
// matches every other tablet's and the master's.
@@ -493,10 +547,20 @@ class Ksck {
private:
friend class KsckTest;
+ // Accumulate information about flags from a server into a FlagToServersMap and
+ // a FlagTagsMap.
+ // 'flags_to_server_map' and 'flag_tags_map' must not be null.
+ void AddFlagsToFlagMaps(const server::GetFlagsResponsePB& flags,
+ const std::string& server_address,
+ KsckFlagToServersMap* flags_to_servers_map,
+ KsckFlagTagsMap* flag_tags_map);
+
bool VerifyTable(const std::shared_ptr<KsckTable>& table);
+
bool VerifyTableWithTimeout(const std::shared_ptr<KsckTable>& table,
const MonoDelta& timeout,
const MonoDelta& retry_interval);
+
KsckCheckResult VerifyTablet(const std::shared_ptr<KsckTablet>& tablet,
int table_num_replicas);
http://git-wip-us.apache.org/repos/asf/kudu/blob/66e41bda/src/kudu/tools/ksck_remote-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/ksck_remote-test.cc b/src/kudu/tools/ksck_remote-test.cc
index b807759..456544f 100644
--- a/src/kudu/tools/ksck_remote-test.cc
+++ b/src/kudu/tools/ksck_remote-test.cc
@@ -24,6 +24,7 @@
#include <vector>
#include <boost/core/ref.hpp>
+#include <gflags/gflags.h>
#include <gflags/gflags_declare.h>
#include <glog/logging.h>
#include <gtest/gtest.h>
@@ -34,6 +35,7 @@
#include "kudu/client/write_op.h"
#include "kudu/common/partial_row.h"
#include "kudu/gutil/gscoped_ptr.h"
+#include "kudu/gutil/macros.h"
#include "kudu/gutil/port.h"
#include "kudu/gutil/ref_counted.h"
#include "kudu/gutil/strings/substitute.h"
@@ -44,6 +46,7 @@
#include "kudu/tserver/mini_tablet_server.h"
#include "kudu/util/atomic.h"
#include "kudu/util/countdown_latch.h"
+#include "kudu/util/flag_tags.h"
#include "kudu/util/monotime.h"
#include "kudu/util/net/net_util.h"
#include "kudu/util/net/sockaddr.h"
@@ -55,6 +58,12 @@
#include "kudu/util/thread.h"
DECLARE_int32(heartbeat_interval_ms);
+DECLARE_int32(safe_time_max_lag_ms);
+
+DEFINE_int32(experimental_flag_for_ksck_test, 0,
+ "A flag marked experimental so it can be used to test ksck's "
+ "unusual flag detection features");
+TAG_FLAG(experimental_flag_for_ksck_test, experimental);
namespace kudu {
namespace tools {
@@ -213,14 +222,36 @@ class RemoteKsckTest : public KuduTest {
TEST_F(RemoteKsckTest, TestClusterOk) {
ASSERT_OK(ksck_->CheckMasterHealth());
+ ASSERT_OK(ksck_->CheckMasterUnusualFlags());
ASSERT_OK(ksck_->CheckMasterConsensus());
ASSERT_OK(ksck_->CheckClusterRunning());
ASSERT_OK(ksck_->FetchTableAndTabletInfo());
ASSERT_OK(ksck_->FetchInfoFromTabletServers());
+ ASSERT_OK(ksck_->CheckTabletServerUnusualFlags());
+}
+
+TEST_F(RemoteKsckTest, TestCheckUnusualFlags) {
+ FLAGS_experimental_flag_for_ksck_test = 1;
+
+ ASSERT_OK(ksck_->CheckMasterHealth());
+ ASSERT_OK(ksck_->CheckMasterUnusualFlags());
+ ASSERT_OK(ksck_->CheckClusterRunning());
+ ASSERT_OK(ksck_->FetchTableAndTabletInfo());
+ ASSERT_OK(ksck_->FetchInfoFromTabletServers());
+ ASSERT_OK(ksck_->CheckTabletServerUnusualFlags());
+ ASSERT_OK(ksck_->PrintResults());
+
+ const string& err_string = err_stream_.str();
+ ASSERT_STR_CONTAINS(err_string,
+ "Some masters have unsafe, experimental, or hidden flags set");
+ ASSERT_STR_CONTAINS(err_string,
+ "Some tablet servers have unsafe, experimental, or hidden flags set");
+ ASSERT_STR_CONTAINS(err_string, "experimental_flag_for_ksck_test");
}
TEST_F(RemoteKsckTest, TestTabletServerMismatchedUUID) {
ASSERT_OK(ksck_->CheckMasterHealth());
+ ASSERT_OK(ksck_->CheckMasterUnusualFlags());
ASSERT_OK(ksck_->CheckMasterConsensus());
ASSERT_OK(ksck_->CheckClusterRunning());
ASSERT_OK(ksck_->FetchTableAndTabletInfo());
@@ -257,6 +288,7 @@ TEST_F(RemoteKsckTest, TestTableConsistency) {
Status s;
while (MonoTime::Now() < deadline) {
ASSERT_OK(ksck_->CheckMasterHealth());
+ ASSERT_OK(ksck_->CheckMasterUnusualFlags());
ASSERT_OK(ksck_->CheckMasterConsensus());
ASSERT_OK(ksck_->CheckClusterRunning());
ASSERT_OK(ksck_->FetchTableAndTabletInfo());
@@ -279,6 +311,7 @@ TEST_F(RemoteKsckTest, TestChecksum) {
Status s;
while (MonoTime::Now() < deadline) {
ASSERT_OK(ksck_->CheckMasterHealth());
+ ASSERT_OK(ksck_->CheckMasterUnusualFlags());
ASSERT_OK(ksck_->CheckMasterConsensus());
ASSERT_OK(ksck_->CheckClusterRunning());
ASSERT_OK(ksck_->FetchTableAndTabletInfo());
@@ -327,6 +360,7 @@ TEST_F(RemoteKsckTest, TestChecksumSnapshot) {
uint64_t ts = client_->GetLatestObservedTimestamp();
ASSERT_OK(ksck_->CheckMasterHealth());
+ ASSERT_OK(ksck_->CheckMasterUnusualFlags());
ASSERT_OK(ksck_->CheckMasterConsensus());
ASSERT_OK(ksck_->CheckClusterRunning());
ASSERT_OK(ksck_->FetchTableAndTabletInfo());
@@ -352,6 +386,7 @@ TEST_F(RemoteKsckTest, TestChecksumSnapshotCurrentTimestamp) {
CHECK(started_writing.WaitFor(MonoDelta::FromSeconds(30)));
ASSERT_OK(ksck_->CheckMasterHealth());
+ ASSERT_OK(ksck_->CheckMasterUnusualFlags());
ASSERT_OK(ksck_->CheckMasterConsensus());
ASSERT_OK(ksck_->CheckClusterRunning());
ASSERT_OK(ksck_->FetchTableAndTabletInfo());
@@ -367,6 +402,7 @@ TEST_F(RemoteKsckTest, TestLeaderMasterDown) {
// Make sure ksck's client is created with the current leader master and that
// all masters are healthy.
ASSERT_OK(ksck_->CheckMasterHealth());
+ ASSERT_OK(ksck_->CheckMasterUnusualFlags());
ASSERT_OK(ksck_->CheckMasterConsensus());
ASSERT_OK(ksck_->CheckClusterRunning());
http://git-wip-us.apache.org/repos/asf/kudu/blob/66e41bda/src/kudu/tools/ksck_remote.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/ksck_remote.cc b/src/kudu/tools/ksck_remote.cc
index 3c92eca..f908136 100644
--- a/src/kudu/tools/ksck_remote.cc
+++ b/src/kudu/tools/ksck_remote.cc
@@ -89,6 +89,18 @@ namespace {
MonoDelta GetDefaultTimeout() {
return MonoDelta::FromMilliseconds(FLAGS_timeout_ms);
}
+
+// Common flag-fetching routine for masters and tablet servers.
+Status FetchUnusualFlagsCommon(const shared_ptr<server::GenericServiceProxy>& proxy,
+ server::GetFlagsResponsePB* resp) {
+ server::GetFlagsRequestPB req;
+ RpcController rpc;
+ rpc.set_timeout(GetDefaultTimeout());
+ for (const string& tag : { "experimental", "hidden", "unsafe" }) {
+ req.add_tags(tag);
+ }
+ return proxy->GetFlags(req, resp, &rpc);
+}
} // anonymous namespace
Status RemoteKsckMaster::Init() {
@@ -140,6 +152,18 @@ Status RemoteKsckMaster::FetchConsensusState() {
return Status::OK();
}
+Status RemoteKsckMaster::FetchUnusualFlags() {
+ server::GetFlagsResponsePB resp;
+ Status s = FetchUnusualFlagsCommon(generic_proxy_, &resp);
+ if (!s.ok()) {
+ flags_state_ = KsckFetchState::FETCH_FAILED;
+ } else {
+ flags_state_ = KsckFetchState::FETCHED;
+ flags_ = resp;
+ }
+ return s;
+}
+
Status RemoteKsckTabletServer::Init() {
vector<Sockaddr> addresses;
RETURN_NOT_OK(ParseAddressList(
@@ -231,6 +255,18 @@ Status RemoteKsckTabletServer::FetchConsensusState(KsckServerHealth* health) {
return Status::OK();
}
+Status RemoteKsckTabletServer::FetchUnusualFlags() {
+ server::GetFlagsResponsePB resp;
+ Status s = FetchUnusualFlagsCommon(generic_proxy_, &resp);
+ if (!s.ok()) {
+ flags_state_ = KsckFetchState::FETCH_FAILED;
+ } else {
+ flags_state_ = KsckFetchState::FETCHED;
+ flags_ = resp;
+ }
+ return s;
+}
+
class ChecksumStepper;
// Simple class to act as a callback in order to collate results from parallel
http://git-wip-us.apache.org/repos/asf/kudu/blob/66e41bda/src/kudu/tools/ksck_remote.h
----------------------------------------------------------------------
diff --git a/src/kudu/tools/ksck_remote.h b/src/kudu/tools/ksck_remote.h
index 0ceda93..07bb0b7 100644
--- a/src/kudu/tools/ksck_remote.h
+++ b/src/kudu/tools/ksck_remote.h
@@ -74,6 +74,8 @@ class RemoteKsckMaster : public KsckMaster {
// Gathers consensus state for the master tablet.
Status FetchConsensusState() override;
+ Status FetchUnusualFlags() override;
+
private:
std::shared_ptr<rpc::Messenger> messenger_;
std::shared_ptr<server::GenericServiceProxy> generic_proxy_;
@@ -99,6 +101,8 @@ class RemoteKsckTabletServer : public KsckTabletServer {
Status FetchConsensusState(KsckServerHealth* health) override;
+ Status FetchUnusualFlags() override;
+
void RunTabletChecksumScanAsync(
const std::string& tablet_id,
const Schema& schema,
http://git-wip-us.apache.org/repos/asf/kudu/blob/66e41bda/src/kudu/tools/ksck_results.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/ksck_results.cc b/src/kudu/tools/ksck_results.cc
index 476db0e..008719d 100644
--- a/src/kudu/tools/ksck_results.cc
+++ b/src/kudu/tools/ksck_results.cc
@@ -19,6 +19,7 @@
#include <algorithm>
#include <iomanip>
#include <iostream>
+#include <iterator>
#include <map>
#include <memory>
#include <numeric>
@@ -28,16 +29,24 @@
#include <unordered_map>
#include <utility>
+#include <gflags/gflags.h>
#include <glog/logging.h>
#include "kudu/gutil/map-util.h"
+#include "kudu/gutil/strings/join.h"
#include "kudu/gutil/strings/substitute.h"
#include "kudu/tools/color.h"
#include "kudu/tools/tool.pb.h"
#include "kudu/tools/tool_action_common.h"
#include "kudu/util/jsonwriter.h"
+#include "kudu/util/slice.h"
#include "kudu/util/status.h"
+DEFINE_uint32(truncate_server_csv_length, 3,
+ "Maximum length of server CSVs before truncation. Raise this to "
+ "see more servers with, e.g., unusual flags, at the cost of "
+ "output with long lines.");
+
using std::endl;
using std::left;
using std::map;
@@ -119,6 +128,28 @@ bool ServerByHealthComparator(const KsckServerHealthSummary& left,
std::make_tuple(ServerHealthScore(right.health), right.uuid, right.address);
}
+// Produces a possibly-abbreviated CSV of 'servers':
+// - If 'servers.size() == server_count', returns a special message indicating
+// all servers.
+// - If servers.size() > FLAGS_truncate_server_csv_length, returns a csv with
+// the first FLAGS_truncate_server_csv_length servers and a final component
+// indicating how many are elided.
+// Requires that 'servers.size() <= server_count'
+string ServerCsv(int server_count, const vector<string>& servers) {
+ DCHECK_LE(servers.size(), server_count);
+ if (servers.size() == server_count) {
+ return Substitute("all $0 server(s) checked", server_count);
+ }
+ uint32_t n = FLAGS_truncate_server_csv_length;
+ if (servers.size() <= n) {
+ return JoinStrings(servers, ", ");
+ }
+ vector<string> first_n;
+ std::copy_n(servers.begin(), n, std::back_inserter(first_n));
+ first_n.push_back(Substitute("and $0 other server(s)", servers.size() - n));
+ return JoinStrings(first_n, ", ");
+}
+
} // anonymous namespace
const char* const KsckCheckResultToString(KsckCheckResult cr) {
@@ -197,6 +228,15 @@ Status KsckResults::PrintTo(PrintMode mode, ostream& out) {
}
out << endl;
+ RETURN_NOT_OK(PrintFlagTable(KsckServerType::MASTER,
+ master_summaries.size(),
+ master_flag_to_servers_map,
+ master_flag_tags_map,
+ out));
+ if (!master_flag_to_servers_map.empty()) {
+ out << endl;
+ }
+
// Then, on the health of the tablet servers.
std::sort(tserver_summaries.begin(), tserver_summaries.end(), ServerByHealthComparator);
RETURN_NOT_OK(PrintServerHealthSummaries(KsckServerType::TABLET_SERVER,
@@ -206,6 +246,15 @@ Status KsckResults::PrintTo(PrintMode mode, ostream& out) {
out << endl;
}
+ RETURN_NOT_OK(PrintFlagTable(KsckServerType::TABLET_SERVER,
+ tserver_summaries.size(),
+ tserver_flag_to_servers_map,
+ tserver_flag_tags_map,
+ out));
+ if (!tserver_flag_to_servers_map.empty()) {
+ out << endl;
+ }
+
// Then, on each tablet.
RETURN_NOT_OK(PrintTabletSummaries(tablet_summaries, mode, out));
@@ -227,6 +276,17 @@ Status KsckResults::PrintTo(PrintMode mode, ostream& out) {
// And, add a summary of all the things we checked.
RETURN_NOT_OK(PrintTotalCounts(*this, out));
+ // Penultimately, print the warnings.
+ if (!warning_messages.empty()) {
+ out << "==================" << endl;
+ out << "Warnings:" << endl;
+ out << "==================" << endl;
+ for (const auto& s : warning_messages) {
+ out << s.message().ToString() << endl;
+ }
+ out << endl;
+ }
+
// Finally, print a summary of all errors.
if (error_messages.empty()) {
// All good.
@@ -305,6 +365,26 @@ Status PrintServerHealthSummaries(KsckServerType type,
return Status::OK();
}
+Status PrintFlagTable(KsckServerType type,
+ int num_servers,
+ const KsckFlagToServersMap& flag_to_servers_map,
+ const KsckFlagTagsMap& flag_tags_map,
+ ostream& out) {
+ if (flag_to_servers_map.empty()) {
+ return Status::OK();
+ }
+ DataTable flags_table({"Flag", "Value", "Tags", ServerTypeToString(type)});
+ for (const auto& flag : flag_to_servers_map) {
+ const string& name = flag.first.first;
+ const string& value = flag.first.second;
+ flags_table.AddRow({name,
+ value,
+ FindOrDie(flag_tags_map, name),
+ ServerCsv(num_servers, flag.second)});
+ }
+ return flags_table.PrintTo(out);
+}
+
Status PrintTableSummaries(const vector<KsckTableSummary>& table_summaries,
ostream& out) {
if (table_summaries.empty()) {
http://git-wip-us.apache.org/repos/asf/kudu/blob/66e41bda/src/kudu/tools/ksck_results.h
----------------------------------------------------------------------
diff --git a/src/kudu/tools/ksck_results.h b/src/kudu/tools/ksck_results.h
index a7b9c02..b42d5aa 100644
--- a/src/kudu/tools/ksck_results.h
+++ b/src/kudu/tools/ksck_results.h
@@ -21,6 +21,7 @@
#include <map>
#include <set>
#include <string>
+#include <unordered_map>
#include <utility>
#include <vector>
@@ -256,6 +257,15 @@ enum class PrintMode {
typedef std::map<std::string, KsckConsensusState> KsckConsensusStateMap;
+// A flag and its value.
+typedef std::pair<std::string, std::string> KsckFlag;
+
+// Map (flag name, flag value) -> server uuids with --flag=value.
+typedef std::map<KsckFlag, std::vector<std::string>> KsckFlagToServersMap;
+
+// Convenience map flag name -> flag tags.
+typedef std::unordered_map<std::string, std::string> KsckFlagTagsMap;
+
// Container for all the results of a series of ksck checks.
struct KsckResults {
// Collection of error status for failed checks. Used to print out a final
@@ -263,10 +273,21 @@ struct KsckResults {
// All checks passed if and only if this vector is empty.
std::vector<Status> error_messages;
+ // Collection of warnings from checks.
+ // These errors are not considered to indicate an unhealthy cluster,
+ // so they do not cause ksck to report an error.
+ std::vector<Status> warning_messages;
+
// Health summaries for master and tablet servers.
std::vector<KsckServerHealthSummary> master_summaries;
std::vector<KsckServerHealthSummary> tserver_summaries;
+ // Information about the flags of masters and tablet servers.
+ KsckFlagToServersMap master_flag_to_servers_map;
+ KsckFlagTagsMap master_flag_tags_map;
+ KsckFlagToServersMap tserver_flag_to_servers_map;
+ KsckFlagTagsMap tserver_flag_tags_map;
+
// Information about the master consensus configuration.
std::vector<std::string> master_uuids;
bool master_consensus_conflict = false;
@@ -296,6 +317,15 @@ Status PrintServerHealthSummaries(KsckServerType type,
const std::vector<KsckServerHealthSummary>& summaries,
std::ostream& out);
+// Print a formatted summary of the flags in 'flag_to_servers_map', indicating
+// which servers have which (flag, value) pairs set.
+// Flag tag information is sourced from 'flag_tags_map'.
+Status PrintFlagTable(KsckServerType type,
+ int num_servers,
+ const KsckFlagToServersMap& flag_to_servers_map,
+ const KsckFlagTagsMap& flag_tags_map,
+ std::ostream& out);
+
// Print a formatted summary of the tables in 'table_summaries' to 'out'.
Status PrintTableSummaries(const std::vector<KsckTableSummary>& table_summaries,
std::ostream& out);
[4/5] kudu git commit: Mark the ImportExportFiles tool unstable
Posted by mp...@apache.org.
Mark the ImportExportFiles tool unstable
Given we did not add an audience interface annotation
and the issues found in KUDU-2454, this change
markes ImportExportFiles.scala as unstable to
discourage it’s use until more fixes and testing can
be done.
It also marks IntegrationTestBigLinkedList.scala private
given it was also missing an audience interface annotation.
Change-Id: I60af80b675e9918ad0e1742076d23d3c00738574
Reviewed-on: http://gerrit.cloudera.org:8080/10507
Tested-by: Kudu Jenkins
Reviewed-by: Hao Hao <ha...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/6662a907
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/6662a907
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/6662a907
Branch: refs/heads/master
Commit: 6662a907b75b3731f5066ae1c2c6592e8197949f
Parents: 66e41bd
Author: Grant Henke <gr...@apache.org>
Authored: Thu May 24 16:45:54 2018 -0500
Committer: Grant Henke <gr...@apache.org>
Committed: Tue May 29 21:31:13 2018 +0000
----------------------------------------------------------------------
.../scala/org/apache/kudu/spark/tools/ImportExportFiles.scala | 4 +++-
.../apache/kudu/spark/tools/IntegrationTestBigLinkedList.scala | 4 +++-
2 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/6662a907/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/ImportExportFiles.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/ImportExportFiles.scala b/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/ImportExportFiles.scala
index e31a058..fc1ef3e 100644
--- a/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/ImportExportFiles.scala
+++ b/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/ImportExportFiles.scala
@@ -25,8 +25,10 @@ import org.apache.spark.sql.SparkSession
import org.apache.spark.SparkConf
import org.slf4j.{Logger, LoggerFactory}
import org.apache.kudu.spark.kudu._
-import org.apache.yetus.audience.InterfaceAudience
+import org.apache.yetus.audience.{InterfaceAudience, InterfaceStability}
+@InterfaceAudience.Public
+@InterfaceStability.Unstable //TODO: Unstable due to KUDU-2454
object ImportExportKudu {
val LOG: Logger = LoggerFactory.getLogger(ImportExportKudu.getClass)
http://git-wip-us.apache.org/repos/asf/kudu/blob/6662a907/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/IntegrationTestBigLinkedList.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/IntegrationTestBigLinkedList.scala b/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/IntegrationTestBigLinkedList.scala
index 9ae6e1c..0b2e4cd 100644
--- a/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/IntegrationTestBigLinkedList.scala
+++ b/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/IntegrationTestBigLinkedList.scala
@@ -25,7 +25,7 @@ import org.apache.kudu.mapreduce.tools.BigLinkedListCommon.{Xoroshiro128PlusRand
import org.apache.kudu.spark.kudu.KuduContext
import org.apache.spark.sql.SparkSession
import org.apache.spark.{SparkConf, TaskContext}
-import org.apache.yetus.audience.InterfaceAudience
+import org.apache.yetus.audience.{InterfaceAudience, InterfaceStability}
import org.slf4j.{Logger, LoggerFactory}
import scala.util.Try
@@ -42,6 +42,8 @@ import scala.util.Try
* This is hopefully easier to understand, and has the advantage of stressing
* slightly different code paths than the MR version.
*/
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
object IntegrationTestBigLinkedList {
val LOG: Logger = LoggerFactory.getLogger(IntegrationTestBigLinkedList.getClass)
[5/5] kudu git commit: KUDU-702 Add block IDs to more log messages
Posted by mp...@apache.org.
KUDU-702 Add block IDs to more log messages
Added block id to the log messages in the following cases:
1. Log Warnings during cfile read
2. VLOG on adding and removing unlocked blocks
Testing was performed by manually inspecting the log message
outputs generated by executing the affected statements.
Change-Id: I895da1cc04ecbf006f412f06b31461b03072d32d
Reviewed-on: http://gerrit.cloudera.org:8080/10505
Tested-by: Kudu Jenkins
Reviewed-by: Mike Percy <mp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/e840ad7e
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/e840ad7e
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/e840ad7e
Branch: refs/heads/master
Commit: e840ad7e914bcfaf188470e35634cbf4f477d148
Parents: 6662a90
Author: anupama <an...@cloudera.com>
Authored: Fri May 25 12:11:14 2018 -0700
Committer: Mike Percy <mp...@apache.org>
Committed: Tue May 29 23:06:20 2018 +0000
----------------------------------------------------------------------
src/kudu/cfile/cfile_reader.cc | 7 ++++---
src/kudu/cfile/index_btree.cc | 3 +--
src/kudu/fs/log_block_manager.cc | 8 ++++----
3 files changed, 9 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/e840ad7e/src/kudu/cfile/cfile_reader.cc
----------------------------------------------------------------------
diff --git a/src/kudu/cfile/cfile_reader.cc b/src/kudu/cfile/cfile_reader.cc
index 60b7d19..3211751 100644
--- a/src/kudu/cfile/cfile_reader.cc
+++ b/src/kudu/cfile/cfile_reader.cc
@@ -488,8 +488,8 @@ Status CFileReader::ReadBlock(const BlockPointer &ptr, CacheControl cache_contro
CompressedBlockDecoder uncompressor(codec_, cfile_version_, block);
Status s = uncompressor.Init();
if (!s.ok()) {
- LOG(WARNING) << "Unable to validate compressed block at "
- << ptr.offset() << " of size " << block.size() << ": "
+ LOG(WARNING) << "Unable to validate compressed block " << block_id().ToString()
+ << " at " << ptr.offset() << " of size " << block.size() << ": "
<< s.ToString();
return s;
}
@@ -505,7 +505,8 @@ Status CFileReader::ReadBlock(const BlockPointer &ptr, CacheControl cache_contro
}
s = uncompressor.UncompressIntoBuffer(decompressed_scratch.get());
if (!s.ok()) {
- LOG(WARNING) << "Unable to uncompress block at " << ptr.offset()
+ LOG(WARNING) << "Unable to uncompress block " << block_id().ToString()
+ << " at " << ptr.offset()
<< " of size " << block.size() << ": " << s.ToString();
return s;
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/e840ad7e/src/kudu/cfile/index_btree.cc
----------------------------------------------------------------------
diff --git a/src/kudu/cfile/index_btree.cc b/src/kudu/cfile/index_btree.cc
index 013496f..019777c 100644
--- a/src/kudu/cfile/index_btree.cc
+++ b/src/kudu/cfile/index_btree.cc
@@ -106,8 +106,7 @@ Status IndexTreeBuilder::Finish(BTreeInfoPB *info) {
BlockPointer ptr;
Status s = FinishAndWriteBlock(root_level, &ptr);
if (!s.ok()) {
- LOG(ERROR) << "Unable to flush root index block";
- return s;
+ return s.CloneAndPrepend("Unable to flush root index block");
}
VLOG(1) << "Flushed root index block: " << ptr.ToString();
http://git-wip-us.apache.org/repos/asf/kudu/blob/e840ad7e/src/kudu/fs/log_block_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/log_block_manager.cc b/src/kudu/fs/log_block_manager.cc
index 7fb63e4..22360da 100644
--- a/src/kudu/fs/log_block_manager.cc
+++ b/src/kudu/fs/log_block_manager.cc
@@ -2038,8 +2038,8 @@ bool LogBlockManager::AddLogBlockUnlocked(scoped_refptr<LogBlock> lb) {
return false;
}
- VLOG(2) << Substitute("Added block: offset $0, length $1",
- lb->offset(), lb->length());
+ VLOG(2) << Substitute("Added block: id $0, offset $1, length $2",
+ lb->block_id().ToString(), lb->offset(), lb->length());
// There may already be an entry in open_block_ids_ (e.g. we just finished
// writing out a block).
@@ -2143,8 +2143,8 @@ Status LogBlockManager::RemoveLogBlockUnlocked(const BlockId& block_id,
*lb = std::move(it->second);
blocks_by_block_id_.erase(it);
- VLOG(2) << Substitute("Removed block: offset $0, length $1",
- (*lb)->offset(), (*lb)->length());
+ VLOG(2) << Substitute("Removed block: id $0, offset $1, length $2",
+ (*lb)->block_id().ToString(), (*lb)->offset(), (*lb)->length());
return Status::OK();
}
[2/5] kudu git commit: KUDU-2444
TabletServerTest.TestTombstonedTabletOnWebUI is flaky
Posted by mp...@apache.org.
KUDU-2444 TabletServerTest.TestTombstonedTabletOnWebUI is flaky
It's possible for a tablet to be RUNNING with working consensus
but still have its "creating" transition live in the tablet
manager. This makes deleting a tablet without retry immediately
after it is created flaky. To fix this for tests using an
in-process tablet server, I added a method to the TSTabletManager
that waits for there to be no tablet transitions in progress.
This method is intended for use only in tests, and is used by the
tablet server test fixture to ensure its single tablet has transitioned
out of the creating state before the actual test runs.
In addition to the original report from tablet_server-test.cc,
I also saw a test in tablet_copy_client_session-itest.cc fail
because of the same issue. I changed it to retry the delete, as
this tests uses an external minicluster.
I ran the affected tests 1000 times with 8 stress threads in
debug mode in dist-test. All passed.
Change-Id: Ib8dc7acc4c39f9d9b067a52df4bd6732f545a8de
Reviewed-on: http://gerrit.cloudera.org:8080/10443
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Will Berkeley <wd...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/98ef3c03
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/98ef3c03
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/98ef3c03
Branch: refs/heads/master
Commit: 98ef3c03b2c07cbbde69c039a0d04ad208358fc0
Parents: e04fd4e
Author: Will Berkeley <wd...@apache.org>
Authored: Thu May 17 13:00:48 2018 -0700
Committer: Will Berkeley <wd...@gmail.com>
Committed: Fri May 25 21:36:31 2018 +0000
----------------------------------------------------------------------
.../tablet_copy_client_session-itest.cc | 9 +++++++--
src/kudu/tserver/tablet_server-test-base.cc | 11 ++++++++---
src/kudu/tserver/tablet_server-test.cc | 11 +++++++----
src/kudu/tserver/ts_tablet_manager.cc | 15 +++++++++++++++
src/kudu/tserver/ts_tablet_manager.h | 5 +++++
5 files changed, 42 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/98ef3c03/src/kudu/integration-tests/tablet_copy_client_session-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/tablet_copy_client_session-itest.cc b/src/kudu/integration-tests/tablet_copy_client_session-itest.cc
index 78a0e77..bacf419 100644
--- a/src/kudu/integration-tests/tablet_copy_client_session-itest.cc
+++ b/src/kudu/integration-tests/tablet_copy_client_session-itest.cc
@@ -269,8 +269,13 @@ TEST_F(TabletCopyClientSessionITest, TestCopyFromCrashedSource) {
// Attempt the copy again. This time it should succeed.
ASSERT_OK(WaitUntilTabletRunning(ts0, tablet_id, kDefaultTimeout));
- ASSERT_OK(StartTabletCopy(ts1, tablet_id, ts0->uuid(), src_addr,
- std::numeric_limits<int64_t>::max(), kDefaultTimeout));
+ // KUDU-2444: Even though the copy failed and the tablet replica is
+ // tombstoned, the tablet manager may not have transitioned the replica out
+ // of the 'copying' phase yet, so we retry here.
+ ASSERT_EVENTUALLY([&]{
+ ASSERT_OK(StartTabletCopy(ts1, tablet_id, ts0->uuid(), src_addr,
+ std::numeric_limits<int64_t>::max(), kDefaultTimeout));
+ });
ASSERT_OK(WaitUntilTabletRunning(ts1, tablet_id, kDefaultTimeout));
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/98ef3c03/src/kudu/tserver/tablet_server-test-base.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_server-test-base.cc b/src/kudu/tserver/tablet_server-test-base.cc
index 1168601..8aaa0dd 100644
--- a/src/kudu/tserver/tablet_server-test-base.cc
+++ b/src/kudu/tserver/tablet_server-test-base.cc
@@ -132,10 +132,15 @@ void TabletServerTestBase::StartTabletServer(int num_data_dirs) {
Status TabletServerTestBase::WaitForTabletRunning(const char *tablet_id) {
scoped_refptr<tablet::TabletReplica> tablet_replica;
- RETURN_NOT_OK(mini_server_->server()->tablet_manager()->GetTabletReplica(tablet_id,
- &tablet_replica));
+ const auto* tablet_manager = mini_server_->server()->tablet_manager();
+ RETURN_NOT_OK(tablet_manager->GetTabletReplica(tablet_id, &tablet_replica));
RETURN_NOT_OK(tablet_replica->WaitUntilConsensusRunning(MonoDelta::FromSeconds(10)));
- return tablet_replica->consensus()->WaitUntilLeaderForTests(MonoDelta::FromSeconds(10));
+ RETURN_NOT_OK(
+ tablet_replica->consensus()->WaitUntilLeaderForTests(MonoDelta::FromSeconds(10)));
+
+ // KUDU-2444: Even though the tablet replica is fully running, the tablet
+ // manager may regard it as still transitioning to the running state.
+ return tablet_manager->WaitForNoTransitionsForTests(MonoDelta::FromSeconds(10));
}
void TabletServerTestBase::UpdateTestRowRemote(int64_t row_idx,
http://git-wip-us.apache.org/repos/asf/kudu/blob/98ef3c03/src/kudu/tserver/tablet_server-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_server-test.cc b/src/kudu/tserver/tablet_server-test.cc
index 436c80f..cc9de59 100644
--- a/src/kudu/tserver/tablet_server-test.cc
+++ b/src/kudu/tserver/tablet_server-test.cc
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-#include "kudu/tserver/tablet_server-test-base.h"
+#include "kudu/tserver/tablet_server.h"
#include <stdlib.h>
#include <unistd.h>
@@ -80,7 +80,7 @@
#include "kudu/tserver/heartbeater.h"
#include "kudu/tserver/mini_tablet_server.h"
#include "kudu/tserver/scanners.h"
-#include "kudu/tserver/tablet_server.h"
+#include "kudu/tserver/tablet_server-test-base.h"
#include "kudu/tserver/tablet_server_options.h"
#include "kudu/tserver/tablet_server_test_util.h"
#include "kudu/tserver/ts_tablet_manager.h"
@@ -536,8 +536,11 @@ TEST_F(TabletServerTest, TestFailedTabletsOnWebUI) {
TEST_F(TabletServerTest, TestTombstonedTabletOnWebUI) {
TSTabletManager* tablet_manager = mini_server_->server()->tablet_manager();
TabletServerErrorPB::Code error_code;
- ASSERT_OK(tablet_manager->DeleteTablet(kTabletId,
- tablet::TABLET_DATA_TOMBSTONED, boost::none, &error_code));
+ ASSERT_OK(
+ tablet_manager->DeleteTablet(kTabletId,
+ tablet::TABLET_DATA_TOMBSTONED,
+ boost::none,
+ &error_code));
// Restart the server. We drop the tablet_replica_ reference since it becomes
// invalid when the server shuts down.
http://git-wip-us.apache.org/repos/asf/kudu/blob/98ef3c03/src/kudu/tserver/ts_tablet_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/ts_tablet_manager.cc b/src/kudu/tserver/ts_tablet_manager.cc
index 59fb0d6..d55b0bb 100644
--- a/src/kudu/tserver/ts_tablet_manager.cc
+++ b/src/kudu/tserver/ts_tablet_manager.cc
@@ -1493,6 +1493,21 @@ int TSTabletManager::RefreshTabletStateCacheAndReturnCount(tablet::TabletStatePB
return FindWithDefault(tablet_state_counts_, st, 0);
}
+Status TSTabletManager::WaitForNoTransitionsForTests(const MonoDelta& timeout) const {
+ const MonoTime start = MonoTime::Now();
+ while (MonoTime::Now() - start < timeout) {
+ {
+ shared_lock<RWMutex> lock(lock_);
+ if (transition_in_progress_.empty()) {
+ return Status::OK();
+ }
+ }
+ SleepFor(MonoDelta::FromMilliseconds(50));
+ }
+ return Status::TimedOut("transitions still in progress after waiting $0",
+ timeout.ToString());
+}
+
TransitionInProgressDeleter::TransitionInProgressDeleter(
TransitionInProgressMap* map, RWMutex* lock, string entry)
: in_progress_(map), lock_(lock), entry_(std::move(entry)) {}
http://git-wip-us.apache.org/repos/asf/kudu/blob/98ef3c03/src/kudu/tserver/ts_tablet_manager.h
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/ts_tablet_manager.h b/src/kudu/tserver/ts_tablet_manager.h
index 742b5a4..30e1b9c 100644
--- a/src/kudu/tserver/ts_tablet_manager.h
+++ b/src/kudu/tserver/ts_tablet_manager.h
@@ -218,6 +218,11 @@ class TSTabletManager : public tserver::TabletReplicaLookupIf {
// and return the count for tablet state 'st'.
int RefreshTabletStateCacheAndReturnCount(tablet::TabletStatePB st);
+ // Wait a period up to 'timeout' for there to be no tablet state transitions
+ // registered with the tablet manager.
+ // This method is for use in tests only. See KUDU-2444.
+ Status WaitForNoTransitionsForTests(const MonoDelta& timeout) const;
+
private:
FRIEND_TEST(TsTabletManagerTest, TestPersistBlocks);