You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by aw...@apache.org on 2018/01/31 21:00:14 UTC
[1/2] kudu git commit: util: move ListFilesInDir to env_util
Repository: kudu
Updated Branches:
refs/heads/master eef0e3942 -> dd57be8c1
util: move ListFilesInDir to env_util
ListFilesInDir() is useful outside of just inspecting external mini
clusters. This patch moves it into env_util.
Change-Id: I8372c13b28c3c3058322fc96b656ef71aa18fc0a
Reviewed-on: http://gerrit.cloudera.org:8080/9166
Tested-by: Kudu Jenkins
Reviewed-by: David Ribeiro Alves <da...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/42711bac
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/42711bac
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/42711bac
Branch: refs/heads/master
Commit: 42711bac4017b3ad11ce4d20ff9e61b6eb0d69f9
Parents: eef0e39
Author: Andrew Wong <aw...@cloudera.com>
Authored: Wed Jan 31 10:31:42 2018 -0800
Committer: Andrew Wong <aw...@cloudera.com>
Committed: Wed Jan 31 20:58:52 2018 +0000
----------------------------------------------------------------------
.../external_mini_cluster_fs_inspector.cc | 26 ++++++--------------
.../external_mini_cluster_fs_inspector.h | 2 --
.../integration-tests/multidir_cluster-itest.cc | 10 ++++----
src/kudu/util/env_util.cc | 15 +++++++++++
src/kudu/util/env_util.h | 5 ++++
5 files changed, 32 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/42711bac/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc b/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc
index bc69a80..911f9ea 100644
--- a/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc
+++ b/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc
@@ -29,6 +29,7 @@
#include "kudu/fs/fs_manager.h"
#include "kudu/mini-cluster/external_mini_cluster.h"
#include "kudu/util/env.h"
+#include "kudu/util/env_util.h"
#include "kudu/util/monotime.h"
#include "kudu/util/path_util.h"
#include "kudu/util/pb_util.h"
@@ -42,6 +43,7 @@ using std::string;
using std::vector;
using cluster::ExternalMiniCluster;
using consensus::ConsensusMetadataPB;
+using env_util::ListFilesInDir;
using strings::Substitute;
using tablet::TabletDataState;
using tablet::TabletSuperBlockPB;
@@ -53,24 +55,10 @@ ExternalMiniClusterFsInspector::ExternalMiniClusterFsInspector(ExternalMiniClust
ExternalMiniClusterFsInspector::~ExternalMiniClusterFsInspector() {}
-Status ExternalMiniClusterFsInspector::ListFilesInDir(const string& path,
- vector<string>* entries) {
- RETURN_NOT_OK(env_->GetChildren(path, entries));
- auto iter = entries->begin();
- while (iter != entries->end()) {
- if (*iter == "." || *iter == ".." || iter->find(kTmpInfix) != string::npos) {
- iter = entries->erase(iter);
- continue;
- }
- ++iter;
- }
- return Status::OK();
-}
-
int ExternalMiniClusterFsInspector::CountFilesInDir(const string& path,
StringPiece pattern) {
vector<string> entries;
- Status s = ListFilesInDir(path, &entries);
+ Status s = ListFilesInDir(env_, path, &entries);
if (!s.ok()) return 0;
return std::count_if(entries.begin(), entries.end(), [&](const string& s) {
return pattern.empty() || MatchPattern(s, pattern);
@@ -81,7 +69,7 @@ int ExternalMiniClusterFsInspector::CountWALFilesOnTS(int index) {
string ts_wal_dir = JoinPathSegments(cluster_->tablet_server(index)->wal_dir(),
FsManager::kWalDirName);
vector<string> tablets;
- CHECK_OK(ListFilesInDir(ts_wal_dir, &tablets));
+ CHECK_OK(ListFilesInDir(env_, ts_wal_dir, &tablets));
int total_segments = 0;
for (const string& tablet : tablets) {
string tablet_wal_dir = JoinPathSegments(ts_wal_dir, tablet);
@@ -103,7 +91,7 @@ vector<string> ExternalMiniClusterFsInspector::ListTabletsOnTS(int index) {
string wal_dir = cluster_->tablet_server(index)->wal_dir();
string meta_dir = JoinPathSegments(wal_dir, FsManager::kTabletMetadataDirName);
vector<string> tablets;
- CHECK_OK(ListFilesInDir(meta_dir, &tablets));
+ CHECK_OK(ListFilesInDir(env_, meta_dir, &tablets));
return tablets;
}
@@ -111,7 +99,7 @@ vector<string> ExternalMiniClusterFsInspector::ListTabletsWithDataOnTS(int index
string wal_dir = JoinPathSegments(cluster_->tablet_server(index)->wal_dir(),
FsManager::kWalDirName);
vector<string> tablets;
- CHECK_OK(ListFilesInDir(wal_dir, &tablets));
+ CHECK_OK(ListFilesInDir(env_, wal_dir, &tablets));
return tablets;
}
@@ -342,7 +330,7 @@ Status ExternalMiniClusterFsInspector::WaitForFilePatternInTabletWalDirOnTs(
string error_msg;
vector<string> entries;
while (true) {
- Status s = ListFilesInDir(tablet_wal_dir, &entries);
+ Status s = ListFilesInDir(env_, tablet_wal_dir, &entries);
std::sort(entries.begin(), entries.end());
error_msg = "";
http://git-wip-us.apache.org/repos/asf/kudu/blob/42711bac/src/kudu/integration-tests/external_mini_cluster_fs_inspector.h
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/external_mini_cluster_fs_inspector.h b/src/kudu/integration-tests/external_mini_cluster_fs_inspector.h
index 3fde5f9..fd82a89 100644
--- a/src/kudu/integration-tests/external_mini_cluster_fs_inspector.h
+++ b/src/kudu/integration-tests/external_mini_cluster_fs_inspector.h
@@ -49,8 +49,6 @@ class ExternalMiniClusterFsInspector {
explicit ExternalMiniClusterFsInspector(cluster::ExternalMiniCluster* cluster);
~ExternalMiniClusterFsInspector();
- Status ListFilesInDir(const std::string& path, std::vector<std::string>* entries);
-
// If provided, files are filtered by the glob-style pattern 'pattern'.
int CountFilesInDir(const std::string& path, StringPiece pattern = StringPiece());
http://git-wip-us.apache.org/repos/asf/kudu/blob/42711bac/src/kudu/integration-tests/multidir_cluster-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/multidir_cluster-itest.cc b/src/kudu/integration-tests/multidir_cluster-itest.cc
index b9d6cb4..25dd253 100644
--- a/src/kudu/integration-tests/multidir_cluster-itest.cc
+++ b/src/kudu/integration-tests/multidir_cluster-itest.cc
@@ -26,11 +26,10 @@
#include "kudu/gutil/map-util.h"
#include "kudu/integration-tests/external_mini_cluster-itest-base.h"
-#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h"
#include "kudu/integration-tests/test_workload.h"
#include "kudu/mini-cluster/external_mini_cluster.h"
+#include "kudu/util/env_util.h"
#include "kudu/util/path_util.h"
-#include "kudu/util/status.h"
#include "kudu/util/test_macros.h"
#include "kudu/util/test_util.h"
@@ -38,6 +37,7 @@ namespace kudu {
using cluster::ExternalMiniClusterOptions;
using cluster::ExternalTabletServer;
+using env_util::ListFilesInDir;
using std::map;
using std::string;
using std::vector;
@@ -74,7 +74,7 @@ TEST_F(MultiDirClusterITest, TestBasicMultiDirCluster) {
for (const string& data_dir : ts->data_dirs()) {
string data_path = JoinPathSegments(data_dir, "data");
vector<string> files;
- ASSERT_OK(inspect_->ListFilesInDir(data_path, &files));
+ ASSERT_OK(ListFilesInDir(env_, data_path, &files));
InsertOrDie(&num_files_in_each_dir, data_dir, files.size());
}
@@ -85,7 +85,7 @@ TEST_F(MultiDirClusterITest, TestBasicMultiDirCluster) {
for (const string& data_dir : ts->data_dirs()) {
string data_path = JoinPathSegments(data_dir, "data");
vector<string> files;
- inspect_->ListFilesInDir(data_path, &files);
+ ListFilesInDir(env_, data_path, &files);
int* num_files_before_insert = FindOrNull(num_files_in_each_dir, data_dir);
ASSERT_NE(nullptr, num_files_before_insert);
if (*num_files_before_insert < files.size()) {
@@ -96,7 +96,7 @@ TEST_F(MultiDirClusterITest, TestBasicMultiDirCluster) {
// data written to it.
ASSERT_GT(num_dirs_added_to, 1);
vector<string> wal_files;
- ASSERT_OK(inspect_->ListFilesInDir(JoinPathSegments(ts->wal_dir(), "wals"), &wal_files));
+ ASSERT_OK(ListFilesInDir(env_, JoinPathSegments(ts->wal_dir(), "wals"), &wal_files));
ASSERT_FALSE(wal_files.empty());
});
work.StopAndJoin();
http://git-wip-us.apache.org/repos/asf/kudu/blob/42711bac/src/kudu/util/env_util.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/env_util.cc b/src/kudu/util/env_util.cc
index 4dbe6d0..dde4138 100644
--- a/src/kudu/util/env_util.cc
+++ b/src/kudu/util/env_util.cc
@@ -301,5 +301,20 @@ Status SyncAllParentDirs(Env* env,
return Status::OK();
}
+Status ListFilesInDir(Env* env,
+ const string& path,
+ vector<string>* entries) {
+ RETURN_NOT_OK(env->GetChildren(path, entries));
+ auto iter = entries->begin();
+ while (iter != entries->end()) {
+ if (*iter == "." || *iter == ".." || iter->find(kTmpInfix) != string::npos) {
+ iter = entries->erase(iter);
+ continue;
+ }
+ ++iter;
+ }
+ return Status::OK();
+}
+
} // namespace env_util
} // namespace kudu
http://git-wip-us.apache.org/repos/asf/kudu/blob/42711bac/src/kudu/util/env_util.h
----------------------------------------------------------------------
diff --git a/src/kudu/util/env_util.h b/src/kudu/util/env_util.h
index bc630e0..de1cc3c 100644
--- a/src/kudu/util/env_util.h
+++ b/src/kudu/util/env_util.h
@@ -101,6 +101,11 @@ Status SyncAllParentDirs(Env* env,
const std::vector<std::string>& dirs,
const std::vector<std::string>& files);
+// Return a list of files within the given 'path'.
+Status ListFilesInDir(Env* env,
+ const std::string& path,
+ std::vector<std::string>* entries);
+
} // namespace env_util
} // namespace kudu
[2/2] kudu git commit: KUDU-1613: evict replicas with wrong server
UUIDs
Posted by aw...@apache.org.
KUDU-1613: evict replicas with wrong server UUIDs
Currently, if a tablet server is wiped and restarted at the same host
and port with a new UUID assigned to it before the replicas on its
previous incarnation are evicted, those replicas won't ever be evicted.
With this patch, upon receiving a response with WRONG_SERVER_UUID,
leaders will treat the follower as failed and handle it appropriately,
re-replicating them elsewhere.
A new test case is added to raft_consensus-itest that is set up to
prompt eviction (i.e. low timeouts, ample servers to replicate to), and
without the change, this test fails.
Change-Id: If0c7ae9dfbd99c2c9a38e8b1f7748c70882e06b4
Reviewed-on: http://gerrit.cloudera.org:8080/9136
Reviewed-by: David Ribeiro Alves <da...@gmail.com>
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/dd57be8c
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/dd57be8c
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/dd57be8c
Branch: refs/heads/master
Commit: dd57be8c1235d5df19ef3d3349b789156b662202
Parents: 42711ba
Author: Andrew Wong <aw...@cloudera.com>
Authored: Tue Jan 23 21:16:54 2018 -0800
Committer: Andrew Wong <aw...@cloudera.com>
Committed: Wed Jan 31 20:59:04 2018 +0000
----------------------------------------------------------------------
src/kudu/consensus/consensus_peers.cc | 20 +++--
.../integration-tests/raft_consensus-itest.cc | 79 ++++++++++++++++++++
src/kudu/mini-cluster/external_mini_cluster.h | 3 +
3 files changed, 95 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/dd57be8c/src/kudu/consensus/consensus_peers.cc
----------------------------------------------------------------------
diff --git a/src/kudu/consensus/consensus_peers.cc b/src/kudu/consensus/consensus_peers.cc
index 187ed6f..26c302a 100644
--- a/src/kudu/consensus/consensus_peers.cc
+++ b/src/kudu/consensus/consensus_peers.cc
@@ -321,13 +321,19 @@ void Peer::ProcessResponse() {
if (response_.has_error()) {
Status response_status = StatusFromPB(response_.error().status());
PeerStatus ps;
- if (response_.error().code() == TabletServerErrorPB::TABLET_FAILED) {
- ps = PeerStatus::TABLET_FAILED;
- } else if (response_.error().code() == TabletServerErrorPB::TABLET_NOT_FOUND) {
- ps = PeerStatus::TABLET_NOT_FOUND;
- } else {
- // Unknown kind of error.
- ps = PeerStatus::REMOTE_ERROR;
+ TabletServerErrorPB resp_error = response_.error();
+ switch (response_.error().code()) {
+ // We treat WRONG_SERVER_UUID as failed.
+ case TabletServerErrorPB::WRONG_SERVER_UUID: FALLTHROUGH_INTENDED;
+ case TabletServerErrorPB::TABLET_FAILED:
+ ps = PeerStatus::TABLET_FAILED;
+ break;
+ case TabletServerErrorPB::TABLET_NOT_FOUND:
+ ps = PeerStatus::TABLET_NOT_FOUND;
+ break;
+ default:
+ // Unknown kind of error.
+ ps = PeerStatus::REMOTE_ERROR;
}
queue_->UpdatePeerStatus(peer_pb_.permanent_uuid(), ps, response_status);
ProcessResponseError(response_status);
http://git-wip-us.apache.org/repos/asf/kudu/blob/dd57be8c/src/kudu/integration-tests/raft_consensus-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/raft_consensus-itest.cc b/src/kudu/integration-tests/raft_consensus-itest.cc
index 4cb0627..4486a63 100644
--- a/src/kudu/integration-tests/raft_consensus-itest.cc
+++ b/src/kudu/integration-tests/raft_consensus-itest.cc
@@ -70,8 +70,12 @@
#include "kudu/tserver/tserver_service.proxy.h"
#include "kudu/util/atomic.h"
#include "kudu/util/countdown_latch.h"
+#include "kudu/util/env.h"
+#include "kudu/util/env_util.h"
#include "kudu/util/metrics.h"
#include "kudu/util/monotime.h"
+#include "kudu/util/net/net_util.h"
+#include "kudu/util/path_util.h"
#include "kudu/util/pb_util.h"
#include "kudu/util/random.h"
#include "kudu/util/scoped_cleanup.h"
@@ -96,7 +100,10 @@ using kudu::client::KuduInsert;
using kudu::client::KuduSession;
using kudu::client::KuduTable;
using kudu::client::sp::shared_ptr;
+using kudu::cluster::ExternalDaemonOptions;
using kudu::cluster::ExternalTabletServer;
+using kudu::cluster::ExternalMiniCluster;
+using kudu::cluster::ExternalMiniClusterOptions;
using kudu::consensus::ConsensusRequestPB;
using kudu::consensus::ConsensusResponsePB;
using kudu::consensus::ConsensusServiceProxy;
@@ -125,6 +132,7 @@ using kudu::itest::WaitUntilTabletInState;
using kudu::itest::WriteSimpleTestRow;
using kudu::master::TabletLocationsPB;
using kudu::master::VOTER_REPLICA;
+using kudu::env_util::ListFilesInDir;
using kudu::pb_util::SecureDebugString;
using kudu::pb_util::SecureShortDebugString;
using kudu::rpc::RpcController;
@@ -2739,5 +2747,76 @@ TEST_F(RaftConsensusITest, TestLogIOErrorIsFatal) {
ASSERT_OK(ext_tservers[0]->WaitForFatal(MonoDelta::FromSeconds(10)));
}
+// KUDU-1613: Test that when we reset and restart a tablet server, with a new
+// uuid but with the same host and port, replicas that were hosted by the
+// previous incarnation are correctly detected as failed and eventually
+// re-replicated.
+TEST_P(RaftConsensusParamReplicationModesITest, TestRestartWithDifferentUUID) {
+ // Start a cluster and insert data.
+ const bool prepare_replacement_before_eviction = GetParam();
+ ExternalMiniClusterOptions opts;
+ opts.num_tablet_servers = 5;
+ opts.extra_tserver_flags = {
+ // Set a low timeout. If we can't re-replicate in a reasonable amount of
+ // time, it means we're not evicting at all.
+ "--follower_unavailable_considered_failed_sec=10",
+ Substitute("--raft_prepare_replacement_before_eviction=$0",
+ prepare_replacement_before_eviction),
+ };
+ opts.extra_master_flags = {
+ Substitute("--raft_prepare_replacement_before_eviction=$0",
+ prepare_replacement_before_eviction),
+ };
+ cluster_.reset(new ExternalMiniCluster(std::move(opts)));
+ ASSERT_OK(cluster_->Start());
+
+ // Write some data. In writing many tablets, we're making it more likely that
+ // all tablet servers will have some tablets on them.
+ TestWorkload writes(cluster_.get());
+ writes.set_num_tablets(25);
+ writes.set_timeout_allowed(true);
+ writes.Setup();
+ writes.Start();
+ const auto wait_for_some_inserts = [&] {
+ const auto rows_init = writes.rows_inserted();
+ ASSERT_EVENTUALLY([&] {
+ ASSERT_GT(writes.rows_inserted() - rows_init, 1000);
+ });
+ };
+ wait_for_some_inserts();
+
+ // Completely shut down one of the tablet servers, keeping its ports so we
+ // can take over with another tablet server.
+ ExternalTabletServer* ts = cluster_->tablet_server(0);
+ vector<HostPort> master_hostports;
+ for (int i = 0; i < cluster_->num_masters(); i++) {
+ master_hostports.push_back(cluster_->master(i)->bound_rpc_hostport());
+ }
+ ExternalDaemonOptions ts_opts = ts->opts();
+ LOG(INFO) << Substitute("Storing port $0 for use by new tablet server",
+ ts->bound_rpc_hostport().ToString());
+ ts_opts.rpc_bind_address = ts->bound_rpc_hostport();
+ ts->Shutdown();
+
+ // With one server down, we should be able to accept writes.
+ wait_for_some_inserts();
+ writes.StopAndJoin();
+
+ // Start up a new server with a new UUID bound to the old RPC port.
+ ts_opts.wal_dir = Substitute("$0-new", ts_opts.wal_dir);
+ ts_opts.data_dirs = { Substitute("$0-new", ts_opts.data_dirs[0]) };
+ ASSERT_OK(env_->CreateDir(ts_opts.wal_dir));
+ ASSERT_OK(env_->CreateDir(ts_opts.data_dirs[0]));
+ scoped_refptr<ExternalTabletServer> new_ts =
+ new ExternalTabletServer(ts_opts, master_hostports);
+ ASSERT_OK(new_ts->Start());
+
+ // Eventually the new server should be copied to.
+ ASSERT_EVENTUALLY([&] {
+ vector<string> files_in_wal_dir;
+ ASSERT_OK(ListFilesInDir(env_, JoinPathSegments(ts_opts.wal_dir, "wals"), &files_in_wal_dir));
+ ASSERT_FALSE(files_in_wal_dir.empty());
+ });
+}
} // namespace tserver
} // namespace kudu
http://git-wip-us.apache.org/repos/asf/kudu/blob/dd57be8c/src/kudu/mini-cluster/external_mini_cluster.h
----------------------------------------------------------------------
diff --git a/src/kudu/mini-cluster/external_mini_cluster.h b/src/kudu/mini-cluster/external_mini_cluster.h
index 17c7c3a..0f0c830 100644
--- a/src/kudu/mini-cluster/external_mini_cluster.h
+++ b/src/kudu/mini-cluster/external_mini_cluster.h
@@ -468,6 +468,9 @@ class ExternalDaemon : public RefCountedThreadSafe<ExternalDaemon> {
// Modifying these flags will only take effect on the next restart.
std::vector<std::string>* mutable_flags() { return &opts_.extra_flags; }
+ // Return the options used to create the daemon.
+ ExternalDaemonOptions opts() const { return opts_; }
+
protected:
friend class RefCountedThreadSafe<ExternalDaemon>;
virtual ~ExternalDaemon();