You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by mp...@apache.org on 2017/09/21 01:13:39 UTC
kudu git commit: consensus: KUDU-2147. Unknown leader should not be
treated as valid UUID
Repository: kudu
Updated Branches:
refs/heads/master 5a0015e75 -> 6b364a24c
consensus: KUDU-2147. Unknown leader should not be treated as valid UUID
This patch fixes a rare, long-standing issue that has existed since at
least 1.4.0, probably much earlier. It appears that the leader election
thread pool changes in 1.5.0 made this problem less rare than it
previously was.
The summary of the issue is that, prior to this fix, it was possible for
the master to believe that no leader existed for a tablet after a
configuration change when in fact a leader did exist. This could be
triggered if the cluster experiences an election storm in the middle or
right after a configuration change. One workaround for this situation is
to restart the tablet server where the leader replica currently resides.
See KUDU-2147 for an example of the error messages that appear in the
logs when it happens.
In addition to a fix, this patch also includes a regression test that
attempts to exercise a code path likely to trigger the bug.
After the fix, I looped the test 200x with 4 stress threads and it
succeeded 100% of the time:
http://dist-test.cloudera.org/job?job_id=mpercy.1505872165.27701
To verify that the issue was not a regression in 1.5.0, I ran it against
the 1.4 branch and it failed 100% of the time:
http://dist-test.cloudera.org/job?job_id=mpercy.1505872429.29509
Change-Id: Ie882d05fc58e55836edc0235d14974e65125df6c
Reviewed-on: http://gerrit.cloudera.org:8080/8109
Reviewed-by: Alexey Serbin <as...@cloudera.com>
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/6b364a24
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/6b364a24
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/6b364a24
Branch: refs/heads/master
Commit: 6b364a24c9b5acff3aa539d0ad8a6ca1ce9d8c67
Parents: 5a0015e
Author: Mike Percy <mp...@apache.org>
Authored: Tue Sep 19 15:11:25 2017 -0700
Committer: Mike Percy <mp...@apache.org>
Committed: Thu Sep 21 01:13:13 2017 +0000
----------------------------------------------------------------------
src/kudu/consensus/consensus_meta-test.cc | 10 +-
src/kudu/consensus/consensus_meta.cc | 4 +-
src/kudu/consensus/quorum_util.cc | 8 +-
src/kudu/integration-tests/CMakeLists.txt | 1 +
.../integration-tests/cluster_itest_util.cc | 2 +-
.../raft_config_change-itest.cc | 158 +++++++++++++++++++
src/kudu/master/catalog_manager.cc | 50 +++---
src/kudu/master/master-path-handlers.cc | 2 +-
src/kudu/tools/ksck.cc | 2 +-
src/kudu/tools/kudu-admin-test.cc | 2 +-
src/kudu/tserver/heartbeater.cc | 14 ++
src/kudu/tserver/ts_tablet_manager-test.cc | 2 +-
12 files changed, 216 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/6b364a24/src/kudu/consensus/consensus_meta-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/consensus/consensus_meta-test.cc b/src/kudu/consensus/consensus_meta-test.cc
index a56b690..3fdc6b6 100644
--- a/src/kudu/consensus/consensus_meta-test.cc
+++ b/src/kudu/consensus/consensus_meta-test.cc
@@ -280,7 +280,15 @@ TEST_F(ConsensusMetadataTest, TestToConsensusStatePB) {
// Set a new leader to be a member of the committed configuration.
cmeta->set_leader_uuid("a");
ConsensusStatePB new_cstate = cmeta->ToConsensusStatePB();
- ASSERT_TRUE(new_cstate.has_leader_uuid());
+ ASSERT_FALSE(new_cstate.leader_uuid().empty());
+ ASSERT_OK(VerifyConsensusState(new_cstate));
+
+ // An empty leader UUID means no leader and we should not set the
+ // corresponding PB field in that case. Regression test for KUDU-2147.
+ cmeta->clear_pending_config();
+ cmeta->set_leader_uuid("");
+ new_cstate = cmeta->ToConsensusStatePB();
+ ASSERT_TRUE(new_cstate.leader_uuid().empty());
ASSERT_OK(VerifyConsensusState(new_cstate));
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/6b364a24/src/kudu/consensus/consensus_meta.cc
----------------------------------------------------------------------
diff --git a/src/kudu/consensus/consensus_meta.cc b/src/kudu/consensus/consensus_meta.cc
index d4702c7..75de554 100644
--- a/src/kudu/consensus/consensus_meta.cc
+++ b/src/kudu/consensus/consensus_meta.cc
@@ -238,7 +238,9 @@ ConsensusStatePB ConsensusMetadata::ToConsensusStatePBUnlocked() const {
lock_.AssertAcquired();
ConsensusStatePB cstate;
cstate.set_current_term(pb_.current_term());
- cstate.set_leader_uuid(leader_uuid_);
+ if (!leader_uuid_.empty()) {
+ cstate.set_leader_uuid(leader_uuid_);
+ }
*cstate.mutable_committed_config() = committed_config_unlocked();
if (has_pending_config_unlocked()) {
*cstate.mutable_pending_config() = pending_config_unlocked();
http://git-wip-us.apache.org/repos/asf/kudu/blob/6b364a24/src/kudu/consensus/quorum_util.cc
----------------------------------------------------------------------
diff --git a/src/kudu/consensus/quorum_util.cc b/src/kudu/consensus/quorum_util.cc
index f4e88d2..6c791b3 100644
--- a/src/kudu/consensus/quorum_util.cc
+++ b/src/kudu/consensus/quorum_util.cc
@@ -73,7 +73,7 @@ Status GetRaftConfigMember(const RaftConfigPB& config,
}
Status GetRaftConfigLeader(const ConsensusStatePB& cstate, RaftPeerPB* peer_pb) {
- if (!cstate.has_leader_uuid() || cstate.leader_uuid().empty()) {
+ if (cstate.leader_uuid().empty()) {
return Status::NotFound("Consensus config has no leader");
}
return GetRaftConfigMember(cstate.committed_config(), cstate.leader_uuid(), peer_pb);
@@ -195,7 +195,7 @@ Status VerifyConsensusState(const ConsensusStatePB& cstate) {
RETURN_NOT_OK(VerifyRaftConfig(cstate.pending_config(), PENDING_CONFIG));
}
- if (cstate.has_leader_uuid() && !cstate.leader_uuid().empty()) {
+ if (!cstate.leader_uuid().empty()) {
if (!IsRaftConfigVoter(cstate.leader_uuid(), cstate.committed_config())
&& cstate.has_pending_config()
&& !IsRaftConfigVoter(cstate.leader_uuid(), cstate.pending_config())) {
@@ -315,13 +315,13 @@ string DiffConsensusStates(const ConsensusStatePB& old_state,
if (leader_changed) {
string old_leader = "<none>";
string new_leader = "<none>";
- if (old_state.has_leader_uuid()) {
+ if (!old_state.leader_uuid().empty()) {
old_leader = Substitute("$0 ($1)",
old_state.leader_uuid(),
committed_peer_infos[old_state.leader_uuid()].first
.last_known_addr().host());
}
- if (new_state.has_leader_uuid()) {
+ if (!new_state.leader_uuid().empty()) {
new_leader = Substitute("$0 ($1)",
new_state.leader_uuid(),
committed_peer_infos[new_state.leader_uuid()].second
http://git-wip-us.apache.org/repos/asf/kudu/blob/6b364a24/src/kudu/integration-tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/CMakeLists.txt b/src/kudu/integration-tests/CMakeLists.txt
index 4326e74..43582d1 100644
--- a/src/kudu/integration-tests/CMakeLists.txt
+++ b/src/kudu/integration-tests/CMakeLists.txt
@@ -86,6 +86,7 @@ ADD_KUDU_TEST(master_replication-itest RESOURCE_LOCK "master-rpc-ports")
ADD_KUDU_TEST(master-stress-test RESOURCE_LOCK "master-rpc-ports")
ADD_KUDU_TEST(multidir_cluster-itest)
ADD_KUDU_TEST(open-readonly-fs-itest)
+ADD_KUDU_TEST(raft_config_change-itest)
ADD_KUDU_TEST(raft_consensus-itest RUN_SERIAL true)
ADD_KUDU_TEST(registration-test RESOURCE_LOCK "master-web-port")
ADD_KUDU_TEST(security-faults-itest)
http://git-wip-us.apache.org/repos/asf/kudu/blob/6b364a24/src/kudu/integration-tests/cluster_itest_util.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/cluster_itest_util.cc b/src/kudu/integration-tests/cluster_itest_util.cc
index cb5a880..326dd53 100644
--- a/src/kudu/integration-tests/cluster_itest_util.cc
+++ b/src/kudu/integration-tests/cluster_itest_util.cc
@@ -510,7 +510,7 @@ Status GetReplicaStatusAndCheckIfLeader(const TServerDetails* replica,
return Status::NotFound("Error connecting to replica", s.ToString());
}
const string& replica_uuid = replica->instance_id.permanent_uuid();
- if (cstate.has_leader_uuid() && cstate.leader_uuid() == replica_uuid) {
+ if (cstate.leader_uuid() == replica_uuid) {
return Status::OK();
}
VLOG(1) << "Replica not leader of config: " << replica->instance_id.permanent_uuid();
http://git-wip-us.apache.org/repos/asf/kudu/blob/6b364a24/src/kudu/integration-tests/raft_config_change-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/raft_config_change-itest.cc b/src/kudu/integration-tests/raft_config_change-itest.cc
new file mode 100644
index 0000000..a7bc9e8
--- /dev/null
+++ b/src/kudu/integration-tests/raft_config_change-itest.cc
@@ -0,0 +1,158 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdint>
+#include <memory>
+#include <ostream>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include <boost/optional/optional.hpp>
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include "kudu/consensus/consensus.pb.h"
+#include "kudu/gutil/strings/substitute.h"
+#include "kudu/integration-tests/cluster_itest_util.h"
+#include "kudu/integration-tests/external_mini_cluster-itest-base.h"
+#include "kudu/integration-tests/external_mini_cluster.h"
+#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h"
+#include "kudu/integration-tests/test_workload.h"
+#include "kudu/master/master.pb.h"
+#include "kudu/util/monotime.h"
+#include "kudu/util/status.h"
+#include "kudu/util/test_macros.h"
+#include "kudu/util/test_util.h"
+
+using kudu::consensus::COMMITTED_OPID;
+using kudu::itest::TServerDetails;
+using std::string;
+using std::vector;
+using strings::Substitute;
+
+namespace kudu {
+
+class RaftConfigChangeITest : public ExternalMiniClusterITestBase {
+};
+
+// Regression test for KUDU-2147. In this test we cause an initial leader to
+// heartbeat to a master with a new configuration change immediately after it
+// has lost its leadership, reporting a new configuration with no leader. The
+// master should update its record of which replica is the leader after a new
+// leader is elected in that term.
+//
+// Steps followed in this test:
+//
+// 1. Inject latency into TS heartbeats to the master and reduce Raft leader
+// heartbeat intervals to followers to be able to carefully control the
+// sequence of events in this test.
+// 2. Evict a follower from the config.
+// 3. Immediately start an election on the remaining follower in the config.
+// We know that this follower should win the election because in order to
+// commit a removal from 3 voters, the removal op must be replicated to both
+// of the two remaining voters in the config.
+// 4. The master will get the config change heartbeat from the initial leader,
+// which will indicate a new term and no current leader.
+// 5. The master will then get a heartbeat from the new leader to report that
+// it's now the leader of the config.
+// 6. Once the master knows who the new leader is, the master will instruct the
+// new leader to add a new replica to its config to bring it back up to 3
+// voters. That new replica will be the follower that we evicted in step 2.
+// 7. If that succeeds then the leader will replicate the eviction, the
+// leader's own no-op, and the adding-back-in of the evicted replica to that
+// evicted replica's log.
+// 8. Once that process completes, all 3 replicas will have identical logs,
+// which is what we wait for at the end of the test.
+TEST_F(RaftConfigChangeITest, TestKudu2147) {
+ if (!AllowSlowTests()) {
+ // This test injects seconds of latency so can take a while to converge.
+ LOG(WARNING) << "Skipping test in fast-test mode.";
+ return;
+ }
+ const MonoDelta kTimeout = MonoDelta::FromSeconds(30);
+ // Slow down leader heartbeats so that in the explicit election below, the
+ // second leader does not immediately heartbeat to the initial leader. If
+ // that occurs, the initial leader will not report to the master that there
+ // is currently no leader.
+ NO_FATALS(StartCluster({"--raft_heartbeat_interval_ms=3000"}));
+
+ // Create a new table.
+ TestWorkload workload(cluster_.get());
+ workload.Setup();
+ workload.Start();
+ ASSERT_EVENTUALLY([&] {
+ ASSERT_GE(workload.batches_completed(), 10);
+ });
+ workload.StopAndJoin();
+
+ // The table should have replicas on three tservers.
+ ASSERT_OK(inspect_->WaitForReplicaCount(3));
+ master::GetTableLocationsResponsePB table_locations;
+ ASSERT_OK(itest::GetTableLocations(cluster_->master_proxy(), TestWorkload::kDefaultTableName,
+ kTimeout, &table_locations));
+ ASSERT_EQ(1, table_locations.tablet_locations_size()); // Only 1 tablet.
+ ASSERT_EQ(3, table_locations.tablet_locations().begin()->replicas_size()); // 3 replicas.
+ string tablet_id = table_locations.tablet_locations().begin()->tablet_id();
+
+ // Wait for all 3 replicas to converge before we start the test.
+ ASSERT_OK(WaitForServersToAgree(kTimeout, ts_map_, tablet_id, 1));
+
+ // Retry as needed to counter normal leader election activity.
+ ASSERT_EVENTUALLY([&] {
+ // Find initial leader.
+ TServerDetails* leader;
+ ASSERT_OK(FindTabletLeader(ts_map_, tablet_id, kTimeout, &leader));
+ ASSERT_OK(WaitForOpFromCurrentTerm(leader, tablet_id, COMMITTED_OPID, kTimeout));
+
+ vector<TServerDetails*> followers;
+ for (int i = 0; i < cluster_->num_tablet_servers(); i++) {
+ // Dynamically set the latency injection flag to induce TS -> master
+ // heartbeat delays. The leader delays its master heartbeats by 2 sec
+ // each time; followers delay their master heartbeats by even longer.
+ int ms_delay = cluster_->tablet_server(i)->uuid() == leader->uuid() ? 2000 : 5000;
+ ASSERT_OK(cluster_->SetFlag(cluster_->tablet_server(i),
+ "heartbeat_inject_latency_before_heartbeat_ms",
+ Substitute("$0", ms_delay)));
+ // Keep track of the followers.
+ if (cluster_->tablet_server(i)->uuid() != leader->uuid()) {
+ followers.push_back(ts_map_[cluster_->tablet_server(i)->uuid()]);
+ }
+ }
+ ASSERT_EQ(2, followers.size());
+
+ // Now that heartbeat injection is enabled, evict one follower and trigger
+ // an election on another follower immediately thereafter.
+ ASSERT_OK(itest::RemoveServer(leader, tablet_id, followers[0],
+ /*cas_config_opid_index=*/ boost::none,
+ kTimeout));
+
+ // Immediately start an election on the remaining follower. This will cause
+ // the initial leader's term to rev and it will have to step down. When it
+ // sends a tablet report to the master with the new configuration excluding
+ // the removed tablet it will report an unknown leader in the new term.
+ ASSERT_OK(itest::StartElection(followers[1], tablet_id, kTimeout));
+ });
+
+ // Wait until the master re-adds the evicted replica and it is back up and
+ // running. If the master hit KUDU-2147, this would fail because the master
+ // would be unable to add the removed server back, and that replica would be
+ // missing the config change op that removed it from the config.
+ ASSERT_OK(WaitForServersToAgree(kTimeout, ts_map_, tablet_id, 1));
+}
+
+} // namespace kudu
http://git-wip-us.apache.org/repos/asf/kudu/blob/6b364a24/src/kudu/master/catalog_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/master/catalog_manager.cc b/src/kudu/master/catalog_manager.cc
index 035a3fe..2b934b2 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -783,7 +783,7 @@ Status CatalogManager::ElectedAsLeaderCb() {
Status CatalogManager::WaitUntilCaughtUpAsLeader(const MonoDelta& timeout) {
ConsensusStatePB cstate = sys_catalog_->tablet_replica()->consensus()->ConsensusState();
const string& uuid = master_->fs_manager()->uuid();
- if (!cstate.has_leader_uuid() || cstate.leader_uuid() != uuid) {
+ if (cstate.leader_uuid() != uuid) {
return Status::IllegalState(
Substitute("Node $0 not leader. Raft Consensus state: $1",
uuid, SecureShortDebugString(cstate)));
@@ -2424,7 +2424,7 @@ bool ShouldTransitionTabletToRunning(const ReportedTabletPB& report) {
// Otherwise, we only transition to RUNNING once there is a leader that is a
// member of the committed configuration.
const ConsensusStatePB& cstate = report.consensus_state();
- return cstate.has_leader_uuid() &&
+ return !cstate.leader_uuid().empty() &&
IsRaftConfigMember(cstate.leader_uuid(), cstate.committed_config());
}
} // anonymous namespace
@@ -2557,9 +2557,9 @@ Status CatalogManager::HandleReportedTablet(TSDescriptor* ts_desc,
return Status::OK();
}
- // If the reported leader is not a member of the committed config, then we
+ // If the reported leader is not a member of the committed config then we
// disregard the leader state.
- if (cstate.has_leader_uuid() &&
+ if (cstate.leader_uuid().empty() ||
!IsRaftConfigMember(cstate.leader_uuid(), cstate.committed_config())) {
cstate.clear_leader_uuid();
}
@@ -2581,10 +2581,10 @@ Status CatalogManager::HandleReportedTablet(TSDescriptor* ts_desc,
"Tablet reported with an active leader");
}
- bool modified_cstate = false;
if (cstate.committed_config().opid_index() > prev_cstate.committed_config().opid_index() ||
- (cstate.has_leader_uuid() &&
- (!prev_cstate.has_leader_uuid() || cstate.current_term() > prev_cstate.current_term()))) {
+ (!cstate.leader_uuid().empty() &&
+ (prev_cstate.leader_uuid().empty() ||
+ cstate.current_term() > prev_cstate.current_term()))) {
// When a config change is reported to the master, it may not include the
// leader because the follower doing the reporting may not know who the
@@ -2593,11 +2593,10 @@ Status CatalogManager::HandleReportedTablet(TSDescriptor* ts_desc,
// previously known for the current term, then retain knowledge of that
// leader even if it wasn't reported in the latest config.
if (cstate.current_term() == prev_cstate.current_term()) {
- if (!cstate.has_leader_uuid() && prev_cstate.has_leader_uuid()) {
+ if (cstate.leader_uuid().empty() && !prev_cstate.leader_uuid().empty()) {
cstate.set_leader_uuid(prev_cstate.leader_uuid());
- modified_cstate = true;
// Sanity check to detect consensus divergence bugs.
- } else if (cstate.has_leader_uuid() && prev_cstate.has_leader_uuid() &&
+ } else if (!cstate.leader_uuid().empty() && !prev_cstate.leader_uuid().empty() &&
cstate.leader_uuid() != prev_cstate.leader_uuid()) {
string msg = Substitute("Previously reported cstate for tablet $0 gave "
"a different leader for term $1 than the current cstate. "
@@ -2612,29 +2611,24 @@ Status CatalogManager::HandleReportedTablet(TSDescriptor* ts_desc,
// If a replica is reporting a new consensus configuration, update the
// master's copy of that configuration.
- LOG(INFO) << Substitute("T $0 reported cstate change: $1. New cstate: $2",
- tablet->tablet_id(), DiffConsensusStates(prev_cstate, cstate),
+ LOG(INFO) << Substitute("T $0 P $1 reported cstate change: $2. New cstate: $3",
+ tablet->tablet_id(), ts_desc->permanent_uuid(),
+ DiffConsensusStates(prev_cstate, cstate),
SecureShortDebugString(cstate));
- // If we need to change the report, copy the whole thing on the stack
- // rather than const-casting.
- const ReportedTabletPB* final_report = &report;
- ReportedTabletPB updated_report;
- if (modified_cstate) {
- updated_report = report;
- *updated_report.mutable_consensus_state() = cstate;
- final_report = &updated_report;
- }
+ // To ensure consistent handling when modifying or sanitizing the cstate
+ // above, copy the whole report on the stack rather than const-casting.
+ ReportedTabletPB modified_report = report;
+ *modified_report.mutable_consensus_state() = cstate;
VLOG(2) << Substitute("Updating cstate for tablet $0 from config reported by $1 "
"to that committed in log index $2 with leader state from term $3",
- final_report->tablet_id(), ts_desc->ToString(),
- final_report->consensus_state().committed_config().opid_index(),
- final_report->consensus_state().current_term());
+ modified_report.tablet_id(), ts_desc->ToString(),
+ modified_report.consensus_state().committed_config().opid_index(),
+ modified_report.consensus_state().current_term());
- RETURN_NOT_OK(HandleRaftConfigChanged(*final_report, tablet,
+ RETURN_NOT_OK(HandleRaftConfigChanged(modified_report, tablet,
table_lock, &tablet_lock));
-
}
}
@@ -2791,7 +2785,7 @@ class PickLeaderReplica : public TSPicker {
// The tablet is still in the PREPARING state and has no replicas.
err_msg = Substitute("Tablet $0 has no consensus state",
tablet_->tablet_id());
- } else if (!l.data().pb.consensus_state().has_leader_uuid()) {
+ } else if (l.data().pb.consensus_state().leader_uuid().empty()) {
// The tablet may be in the midst of a leader election.
err_msg = Substitute("Tablet $0 consensus state has no leader",
tablet_->tablet_id());
@@ -4215,7 +4209,7 @@ CatalogManager::ScopedLeaderSharedLock::ScopedLeaderSharedLock(
consensus()->ConsensusState();
initial_term_ = cstate.current_term();
const string& uuid = catalog_->master_->fs_manager()->uuid();
- if (PREDICT_FALSE(!cstate.has_leader_uuid() || cstate.leader_uuid() != uuid)) {
+ if (PREDICT_FALSE(cstate.leader_uuid() != uuid)) {
leader_status_ = Status::IllegalState(
Substitute("Not the leader. Local UUID: $0, Raft Consensus state: $1",
uuid, SecureShortDebugString(cstate)));
http://git-wip-us.apache.org/repos/asf/kudu/blob/6b364a24/src/kudu/master/master-path-handlers.cc
----------------------------------------------------------------------
diff --git a/src/kudu/master/master-path-handlers.cc b/src/kudu/master/master-path-handlers.cc
index b4fa7c8..a497f9e 100644
--- a/src/kudu/master/master-path-handlers.cc
+++ b/src/kudu/master/master-path-handlers.cc
@@ -489,7 +489,7 @@ class JsonDumper : public TableVisitor, public TabletVisitor {
}
jw_->EndArray();
- if (cs.has_leader_uuid()) {
+ if (!cs.leader_uuid().empty()) {
jw_->String("leader");
jw_->String(cs.leader_uuid());
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/6b364a24/src/kudu/tools/ksck.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/ksck.cc b/src/kudu/tools/ksck.cc
index eb376c4..9913175 100644
--- a/src/kudu/tools/ksck.cc
+++ b/src/kudu/tools/ksck.cc
@@ -729,7 +729,7 @@ Ksck::CheckResult Ksck::VerifyTablet(const shared_ptr<KsckTablet>& tablet, int t
opid_index = config.opid_index();
}
boost::optional<string> repl_leader_uuid;
- if (cstate.has_leader_uuid()) {
+ if (!cstate.leader_uuid().empty()) {
repl_leader_uuid = cstate.leader_uuid();
}
KsckConsensusConfigType config_type = cstate.has_pending_config() ?
http://git-wip-us.apache.org/repos/asf/kudu/blob/6b364a24/src/kudu/tools/kudu-admin-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/kudu-admin-test.cc b/src/kudu/tools/kudu-admin-test.cc
index 44e2447..c36734b 100644
--- a/src/kudu/tools/kudu-admin-test.cc
+++ b/src/kudu/tools/kudu-admin-test.cc
@@ -1065,7 +1065,7 @@ Status GetTermFromConsensus(const vector<TServerDetails*>& tservers,
for (auto& ts : tservers) {
RETURN_NOT_OK(
GetConsensusState(ts, tablet_id, MonoDelta::FromSeconds(10), &cstate));
- if (cstate.has_leader_uuid() &&
+ if (!cstate.leader_uuid().empty() &&
IsRaftConfigMember(cstate.leader_uuid(), cstate.committed_config()) &&
cstate.has_current_term()) {
*current_term = cstate.current_term();
http://git-wip-us.apache.org/repos/asf/kudu/blob/6b364a24/src/kudu/tserver/heartbeater.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/heartbeater.cc b/src/kudu/tserver/heartbeater.cc
index 7ecfd90..012f8df 100644
--- a/src/kudu/tserver/heartbeater.cc
+++ b/src/kudu/tserver/heartbeater.cc
@@ -61,6 +61,7 @@
#include "kudu/util/pb_util.h"
#include "kudu/util/status.h"
#include "kudu/util/thread.h"
+#include "kudu/util/trace.h"
#include "kudu/util/version_info.h"
DEFINE_int32(heartbeat_rpc_timeout_ms, 15000,
@@ -77,6 +78,12 @@ DEFINE_int32(heartbeat_max_failures_before_backoff, 3,
"rather than retrying.");
TAG_FLAG(heartbeat_max_failures_before_backoff, advanced);
+DEFINE_int32(heartbeat_inject_latency_before_heartbeat_ms, 0,
+ "How much latency (in ms) to inject when a tablet copy session is initialized. "
+ "(For testing only!)");
+TAG_FLAG(heartbeat_inject_latency_before_heartbeat_ms, runtime);
+TAG_FLAG(heartbeat_inject_latency_before_heartbeat_ms, unsafe);
+
using kudu::master::MasterServiceProxy;
using kudu::master::TabletReportPB;
using kudu::pb_util::SecureDebugString;
@@ -374,6 +381,13 @@ Status Heartbeater::Thread::DoHeartbeat() {
CHECK(IsCurrentThread());
+ // Inject latency for testing purposes.
+ if (PREDICT_FALSE(FLAGS_heartbeat_inject_latency_before_heartbeat_ms > 0)) {
+ TRACE("Injecting $0ms of latency due to --heartbeat_inject_latency_before_heartbeat_ms",
+ FLAGS_heartbeat_inject_latency_before_heartbeat_ms);
+ SleepFor(MonoDelta::FromMilliseconds(FLAGS_heartbeat_inject_latency_before_heartbeat_ms));
+ }
+
if (!proxy_) {
VLOG(1) << "No valid master proxy. Connecting...";
RETURN_NOT_OK(ConnectToMaster());
http://git-wip-us.apache.org/repos/asf/kudu/blob/6b364a24/src/kudu/tserver/ts_tablet_manager-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/ts_tablet_manager-test.cc b/src/kudu/tserver/ts_tablet_manager-test.cc
index 84dc5f5..47b461f 100644
--- a/src/kudu/tserver/ts_tablet_manager-test.cc
+++ b/src/kudu/tserver/ts_tablet_manager-test.cc
@@ -184,7 +184,7 @@ static void AssertReportHasUpdatedTablet(const TabletReportPB& report,
ASSERT_TRUE(reported_tablet.has_consensus_state());
ASSERT_TRUE(reported_tablet.consensus_state().has_current_term())
<< SecureShortDebugString(reported_tablet);
- ASSERT_TRUE(reported_tablet.consensus_state().has_leader_uuid())
+ ASSERT_FALSE(reported_tablet.consensus_state().leader_uuid().empty())
<< SecureShortDebugString(reported_tablet);
ASSERT_TRUE(reported_tablet.consensus_state().has_committed_config());
const RaftConfigPB& committed_config = reported_tablet.consensus_state().committed_config();