You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by mp...@apache.org on 2017/06/06 01:38:37 UTC
kudu git commit: external minicluster: expand EMC dir usage
Repository: kudu
Updated Branches:
refs/heads/master f329e089f -> fc98b1ae7
external minicluster: expand EMC dir usage
In order to test different disk configurations, it is becoming
increasingly important to have end-to-end testing with nodes backed
by multiple directories.
External miniclusters by default use a single directory for each
daemon's data (i.e. wals and data dirs fall under a single
/cluster/daemon/data directory). This patch adds multi-directory support
via a new 'num_data_dirs' parameter to ExternalMiniClusterOptions.
Additionally, a 'wal_dir' parameter is added to ExternalDaemonOptions to
separate the wal location from the data directories.
If 'num_data_dirs' is greater than 1, each daemon will generate multiple
paths, appending each with a numeric suffix, up to the number specified.
E.g. EMCs that would have used the path /cluster/data, if specifying
'num_data_dirs' as 3, will now generate multiple data directories
/cluster/data-0, /cluster/data-1, /cluster/data-2. The wal will be added
to /cluster/wal.
The new test multidir-cluster-itest demonstrates this.
This test has been run via dist-test 2000 with no flakes. Results here:
http://dist-test.cloudera.org/job?job_id=awong.1496701042.24862
Change-Id: Id2f5def6980ad394c8558ad97ba830f1b0257332
Reviewed-on: http://gerrit.cloudera.org:8080/6845
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Kudu Jenkins
Reviewed-by: Todd Lipcon <to...@apache.org>
Reviewed-by: Mike Percy <mp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/fc98b1ae
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/fc98b1ae
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/fc98b1ae
Branch: refs/heads/master
Commit: fc98b1ae77f6fd869863191fcb7fa5d5363d9fc5
Parents: f329e08
Author: Andrew Wong <aw...@cloudera.com>
Authored: Tue May 30 11:47:05 2017 -0700
Committer: Mike Percy <mp...@apache.org>
Committed: Tue Jun 6 01:38:15 2017 +0000
----------------------------------------------------------------------
src/kudu/integration-tests/CMakeLists.txt | 1 +
.../integration-tests/delete_table-itest.cc | 2 +-
.../integration-tests/disk_reservation-itest.cc | 18 ++--
.../external_mini_cluster-itest-base.cc | 4 +-
.../external_mini_cluster-itest-base.h | 3 +-
.../integration-tests/external_mini_cluster.cc | 67 ++++++++++----
.../integration-tests/external_mini_cluster.h | 44 ++++++++--
.../external_mini_cluster_fs_inspector.cc | 16 ++--
src/kudu/integration-tests/log_verifier.cc | 6 +-
.../integration-tests/master_failover-itest.cc | 13 ++-
.../integration-tests/master_migration-itest.cc | 11 ++-
.../integration-tests/multidir_cluster-itest.cc | 92 ++++++++++++++++++++
.../integration-tests/open-readonly-fs-itest.cc | 4 +-
src/kudu/integration-tests/ts_itest-base.h | 4 +-
src/kudu/integration-tests/ts_recovery-itest.cc | 5 +-
15 files changed, 230 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/CMakeLists.txt b/src/kudu/integration-tests/CMakeLists.txt
index f3a3cc9..07b00a1 100644
--- a/src/kudu/integration-tests/CMakeLists.txt
+++ b/src/kudu/integration-tests/CMakeLists.txt
@@ -81,6 +81,7 @@ ADD_KUDU_TEST_DEPENDENCIES(master_migration-itest
kudu)
ADD_KUDU_TEST(master_replication-itest RESOURCE_LOCK "master-rpc-ports")
ADD_KUDU_TEST(master-stress-test RESOURCE_LOCK "master-rpc-ports")
+ADD_KUDU_TEST(multidir_cluster-itest)
ADD_KUDU_TEST(open-readonly-fs-itest)
ADD_KUDU_TEST(raft_consensus-itest RUN_SERIAL true)
ADD_KUDU_TEST(registration-test RESOURCE_LOCK "master-web-port")
http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/delete_table-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/delete_table-itest.cc b/src/kudu/integration-tests/delete_table-itest.cc
index 9c78054..9419aa7 100644
--- a/src/kudu/integration-tests/delete_table-itest.cc
+++ b/src/kudu/integration-tests/delete_table-itest.cc
@@ -1060,7 +1060,7 @@ TEST_F(DeleteTableITest, TestUnknownTabletsAreNotDeleted) {
// Delete the master's metadata and start it back up. The tablet created
// above is now unknown, but should not be deleted!
cluster_->master()->Shutdown();
- ASSERT_OK(env_->DeleteRecursively(cluster_->master()->data_dir()));
+ ASSERT_OK(cluster_->master()->DeleteFromDisk());
ASSERT_OK(cluster_->master()->Restart());
// Give the master a chance to finish writing the new master tablet to disk
http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/disk_reservation-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/disk_reservation-itest.cc b/src/kudu/integration-tests/disk_reservation-itest.cc
index d30f2e5..b19ffa4 100644
--- a/src/kudu/integration-tests/disk_reservation-itest.cc
+++ b/src/kudu/integration-tests/disk_reservation-itest.cc
@@ -59,13 +59,15 @@ TEST_F(DiskReservationITest, TestFillMultipleDisks) {
// Reserve one byte so that when we simulate 0 bytes free below, we'll start
// failing requests.
ts_flags.push_back("--fs_data_dirs_reserved_bytes=1");
- ts_flags.push_back(Substitute("--fs_data_dirs=$0/a,$0/b", test_dir_));
- ts_flags.push_back(Substitute("--disk_reserved_override_prefix_1_path_for_testing=$0/a",
- test_dir_));
- ts_flags.push_back(Substitute("--disk_reserved_override_prefix_2_path_for_testing=$0/b",
- test_dir_));
- NO_FATALS(StartCluster(ts_flags, {}, 1));
+ NO_FATALS(StartCluster(ts_flags, {}, /* num_tablet_servers= */ 1, /* num_data_dirs= */ 2));
+
+ ASSERT_OK(cluster_->SetFlag(cluster_->tablet_server(0),
+ "disk_reserved_override_prefix_1_path_for_testing",
+ cluster_->GetDataPath("ts-0", 0)));
+ ASSERT_OK(cluster_->SetFlag(cluster_->tablet_server(0),
+ "disk_reserved_override_prefix_2_path_for_testing",
+ cluster_->GetDataPath("ts-0", 1)));
TestWorkload workload(cluster_.get());
workload.set_num_replicas(1);
@@ -77,10 +79,10 @@ TEST_F(DiskReservationITest, TestFillMultipleDisks) {
workload.Setup();
workload.Start();
- // Simulate that /a has 0 bytes free.
+ // Simulate that /data-0 has 0 bytes free.
ASSERT_OK(cluster_->SetFlag(cluster_->tablet_server(0),
"disk_reserved_override_prefix_1_bytes_free_for_testing", "0"));
- // Simulate that /b has 1GB free.
+ // Simulate that /data-1 has 1GB free.
ASSERT_OK(cluster_->SetFlag(cluster_->tablet_server(0),
"disk_reserved_override_prefix_2_bytes_free_for_testing",
Substitute("$0", 1L * 1024 * 1024 * 1024)));
http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/external_mini_cluster-itest-base.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/external_mini_cluster-itest-base.cc b/src/kudu/integration-tests/external_mini_cluster-itest-base.cc
index f4db17f..6b5eefb 100644
--- a/src/kudu/integration-tests/external_mini_cluster-itest-base.cc
+++ b/src/kudu/integration-tests/external_mini_cluster-itest-base.cc
@@ -40,8 +40,10 @@ void ExternalMiniClusterITestBase::TearDown() {
void ExternalMiniClusterITestBase::StartCluster(
const std::vector<std::string>& extra_ts_flags,
const std::vector<std::string>& extra_master_flags,
- int num_tablet_servers) {
+ int num_tablet_servers,
+ int num_data_dirs) {
ExternalMiniClusterOptions opts;
+ opts.num_data_dirs = num_data_dirs;
opts.num_tablet_servers = num_tablet_servers;
opts.extra_master_flags = extra_master_flags;
opts.extra_tserver_flags = extra_ts_flags;
http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/external_mini_cluster-itest-base.h
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/external_mini_cluster-itest-base.h b/src/kudu/integration-tests/external_mini_cluster-itest-base.h
index 87e4750..9032c1a 100644
--- a/src/kudu/integration-tests/external_mini_cluster-itest-base.h
+++ b/src/kudu/integration-tests/external_mini_cluster-itest-base.h
@@ -46,7 +46,8 @@ class ExternalMiniClusterITestBase : public KuduTest {
protected:
void StartCluster(const std::vector<std::string>& extra_ts_flags = {},
const std::vector<std::string>& extra_master_flags = {},
- int num_tablet_servers = 3);
+ int num_tablet_servers = 3,
+ int num_data_dirs = 1);
void StartClusterWithOpts(ExternalMiniClusterOptions opts);
http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/external_mini_cluster.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/external_mini_cluster.cc b/src/kudu/integration-tests/external_mini_cluster.cc
index 79143ea..85f28dc 100644
--- a/src/kudu/integration-tests/external_mini_cluster.cc
+++ b/src/kudu/integration-tests/external_mini_cluster.cc
@@ -93,6 +93,7 @@ ExternalMiniClusterOptions::ExternalMiniClusterOptions()
: num_masters(1),
num_tablet_servers(1),
bind_mode(kBindMode),
+ num_data_dirs(1),
enable_kerberos(false),
logtostderr(true),
start_process_timeout(MonoDelta::FromSeconds(30)) {
@@ -241,14 +242,38 @@ string ExternalMiniCluster::GetBinaryPath(const string& binary) const {
return JoinPathSegments(daemon_bin_path_, binary);
}
-string ExternalMiniCluster::GetDataPath(const string& daemon_id) const {
+string ExternalMiniCluster::GetLogPath(const string& daemon_id) const {
CHECK(!data_root_.empty());
- return JoinPathSegments(JoinPathSegments(data_root_, daemon_id), "data");
+ return JoinPathSegments(JoinPathSegments(data_root_, daemon_id), "logs");
}
-string ExternalMiniCluster::GetLogPath(const string& daemon_id) const {
+string ExternalMiniCluster::GetDataPath(const string& daemon_id,
+ boost::optional<uint32_t> dir_index) const {
CHECK(!data_root_.empty());
- return JoinPathSegments(JoinPathSegments(data_root_, daemon_id), "logs");
+ string data_path = "data";
+ if (dir_index) {
+ CHECK_LT(*dir_index, opts_.num_data_dirs);
+ data_path = Substitute("$0-$1", data_path, dir_index.get());
+ } else {
+ CHECK_EQ(1, opts_.num_data_dirs);
+ }
+ return JoinPathSegments(JoinPathSegments(data_root_, daemon_id), data_path);
+}
+
+vector<string> ExternalMiniCluster::GetDataPaths(const string& daemon_id) const {
+ if (opts_.num_data_dirs == 1) {
+ return { GetDataPath(daemon_id) };
+ }
+ vector<string> paths;
+ for (uint32_t dir_index = 0; dir_index < opts_.num_data_dirs; dir_index++) {
+ paths.emplace_back(GetDataPath(daemon_id, dir_index));
+ }
+ return paths;
+}
+
+string ExternalMiniCluster::GetWalPath(const string& daemon_id) const {
+ CHECK(!data_root_.empty());
+ return JoinPathSegments(JoinPathSegments(data_root_, daemon_id), "wal");
}
namespace {
@@ -270,7 +295,8 @@ Status ExternalMiniCluster::StartSingleMaster() {
ExternalDaemonOptions opts(opts_.logtostderr);
opts.messenger = messenger_;
opts.exe = GetBinaryPath(kMasterBinaryName);
- opts.data_dir = GetDataPath(daemon_id);
+ opts.wal_dir = GetWalPath(daemon_id);
+ opts.data_dirs = GetDataPaths(daemon_id);
opts.log_dir = GetLogPath(daemon_id);
if (FLAGS_perf_record) {
opts.perf_record_filename =
@@ -313,7 +339,8 @@ Status ExternalMiniCluster::StartDistributedMasters() {
ExternalDaemonOptions opts(opts_.logtostderr);
opts.messenger = messenger_;
opts.exe = exe;
- opts.data_dir = GetDataPath(daemon_id);
+ opts.wal_dir = GetWalPath(daemon_id);
+ opts.data_dirs = GetDataPaths(daemon_id);
opts.log_dir = GetLogPath(daemon_id);
if (FLAGS_perf_record) {
opts.perf_record_filename =
@@ -365,7 +392,8 @@ Status ExternalMiniCluster::AddTabletServer() {
ExternalDaemonOptions opts(opts_.logtostderr);
opts.messenger = messenger_;
opts.exe = GetBinaryPath(kTabletServerBinaryName);
- opts.data_dir = GetDataPath(daemon_id);
+ opts.wal_dir = GetWalPath(daemon_id);
+ opts.data_dirs = GetDataPaths(daemon_id);
opts.log_dir = GetLogPath(daemon_id);
if (FLAGS_perf_record) {
opts.perf_record_filename =
@@ -617,7 +645,8 @@ Status ExternalMiniCluster::SetFlag(ExternalDaemon* daemon,
ExternalDaemon::ExternalDaemon(ExternalDaemonOptions opts)
: messenger_(std::move(opts.messenger)),
- data_dir_(std::move(opts.data_dir)),
+ wal_dir_(std::move(opts.wal_dir)),
+ data_dirs_(std::move(opts.data_dirs)),
log_dir_(std::move(opts.log_dir)),
perf_record_filename_(std::move(opts.perf_record_filename)),
start_process_timeout_(opts.start_process_timeout),
@@ -689,7 +718,7 @@ Status ExternalDaemon::StartProcess(const vector<string>& user_flags) {
RETURN_NOT_OK(env_util::CreateDirsRecursively(Env::Default(), log_dir_));
// Tell the server to dump its port information so we can pick it up.
- string info_path = JoinPathSegments(data_dir_, "info.pb");
+ string info_path = JoinPathSegments(data_dirs_[0], "info.pb");
argv.push_back("--server_dump_info_path=" + info_path);
argv.push_back("--server_dump_info_format=pb");
@@ -909,6 +938,14 @@ void ExternalDaemon::Shutdown() {
perf_record_process_.reset();
}
+Status ExternalDaemon::DeleteFromDisk() const {
+ for (const string& data_dir : data_dirs()) {
+ RETURN_NOT_OK(Env::Default()->DeleteRecursively(data_dir));
+ }
+ RETURN_NOT_OK(Env::Default()->DeleteRecursively(wal_dir()));
+ return Status::OK();
+}
+
void ExternalDaemon::FlushCoverage() {
#ifndef COVERAGE_BUILD
return;
@@ -1144,8 +1181,8 @@ Status ExternalMaster::WaitForCatalogManager() {
vector<string> ExternalMaster::GetCommonFlags() const {
return {
- "--fs_wal_dir=" + data_dir_,
- "--fs_data_dirs=" + data_dir_,
+ "--fs_wal_dir=" + wal_dir_,
+ "--fs_data_dirs=" + JoinStrings(data_dirs_, ","),
"--webserver_interface=localhost",
// See the in-line comment for "--ipki_server_key_size" flag in
@@ -1176,8 +1213,8 @@ ExternalTabletServer::~ExternalTabletServer() {
Status ExternalTabletServer::Start() {
vector<string> flags;
- flags.push_back("--fs_wal_dir=" + data_dir_);
- flags.push_back("--fs_data_dirs=" + data_dir_);
+ flags.push_back("--fs_wal_dir=" + wal_dir_);
+ flags.push_back("--fs_data_dirs=" + JoinStrings(data_dirs_, ","));
flags.push_back(Substitute("--rpc_bind_addresses=$0:0",
get_rpc_bind_address()));
flags.push_back(Substitute("--local_ip_for_outbound_sockets=$0",
@@ -1196,8 +1233,8 @@ Status ExternalTabletServer::Restart() {
return Status::IllegalState("Tablet server cannot be restarted. Must call Shutdown() first.");
}
vector<string> flags;
- flags.push_back("--fs_wal_dir=" + data_dir_);
- flags.push_back("--fs_data_dirs=" + data_dir_);
+ flags.push_back("--fs_wal_dir=" + wal_dir_);
+ flags.push_back("--fs_data_dirs=" + JoinStrings(data_dirs_, ","));
flags.push_back("--rpc_bind_addresses=" + bound_rpc_.ToString());
flags.push_back(Substitute("--local_ip_for_outbound_sockets=$0",
get_rpc_bind_address()));
http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/external_mini_cluster.h
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/external_mini_cluster.h b/src/kudu/integration-tests/external_mini_cluster.h
index 0f8d3b3..e8d25c8 100644
--- a/src/kudu/integration-tests/external_mini_cluster.h
+++ b/src/kudu/integration-tests/external_mini_cluster.h
@@ -14,16 +14,18 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-#ifndef KUDU_INTEGRATION_TESTS_EXTERNAL_MINI_CLUSTER_H
-#define KUDU_INTEGRATION_TESTS_EXTERNAL_MINI_CLUSTER_H
+#pragma once
+
+#include <sys/types.h>
#include <functional>
#include <map>
#include <memory>
#include <string>
-#include <sys/types.h>
#include <vector>
+#include <boost/optional.hpp>
+
#include "kudu/client/client.h"
#include "kudu/gutil/gscoped_ptr.h"
#include "kudu/gutil/macros.h"
@@ -110,6 +112,10 @@ struct ExternalMiniClusterOptions {
// This works for unit tests, since they all end up in build/latest/bin.
std::string daemon_bin_path;
+ // Number of data directories to be created for each daemon.
+ // Default: 1
+ int num_data_dirs;
+
// Extra flags for tablet servers and masters respectively.
//
// In these flags, you may use the special string '${index}' which will
@@ -289,7 +295,18 @@ class ExternalMiniCluster : public MiniClusterBase {
// Returns the path where 'daemon_id' is expected to store its data, based on
// ExternalMiniClusterOptions.data_root if it was provided, or on the
// standard Kudu test directory otherwise.
- std::string GetDataPath(const std::string& daemon_id) const;
+ // 'dir_index' is an optional numeric suffix to be added to the default path.
+ // If it is not specified, the cluster must be configured to use a single data dir.
+ std::string GetDataPath(const std::string& daemon_id,
+ boost::optional<uint32_t> dir_index = boost::none) const;
+
+ // Returns paths where 'daemon_id' is expected to store its data, each with a
+ // numeric suffix appropriate for 'opts_.num_data_dirs'
+ std::vector<std::string> GetDataPaths(const std::string& daemon_id) const;
+
+ // Returns the path where 'daemon_id' is expected to store its wal, or other
+ // files that reside in the wal dir.
+ std::string GetWalPath(const std::string& daemon_id) const;
// Returns the path where 'daemon_id' is expected to store its logs, or other
// files that reside in the log dir.
@@ -329,7 +346,8 @@ struct ExternalDaemonOptions {
bool logtostderr;
std::shared_ptr<rpc::Messenger> messenger;
std::string exe;
- std::string data_dir;
+ std::string wal_dir;
+ std::vector<std::string> data_dirs;
std::string log_dir;
std::string perf_record_filename;
std::vector<std::string> extra_flags;
@@ -402,7 +420,17 @@ class ExternalDaemon : public RefCountedThreadSafe<ExternalDaemon> {
virtual void Shutdown();
- const std::string& data_dir() const { return data_dir_; }
+ // Delete files specified by 'wal_dir_' and 'data_dirs_'.
+ Status DeleteFromDisk() const WARN_UNUSED_RESULT;
+
+ const std::string& wal_dir() const { return wal_dir_; }
+
+ const std::string& data_dir() const {
+ CHECK_EQ(1, data_dirs_.size());
+ return data_dirs_[0];
+ }
+
+ const std::vector<std::string>& data_dirs() const { return data_dirs_; }
// Returns the log dir of the external daemon.
const std::string& log_dir() const { return log_dir_; }
@@ -461,7 +489,8 @@ class ExternalDaemon : public RefCountedThreadSafe<ExternalDaemon> {
}
const std::shared_ptr<rpc::Messenger> messenger_;
- const std::string data_dir_;
+ const std::string wal_dir_;
+ std::vector<std::string> data_dirs_;
const std::string log_dir_;
const std::string perf_record_filename_;
const MonoDelta start_process_timeout_;
@@ -547,4 +576,3 @@ class ExternalTabletServer : public ExternalDaemon {
};
} // namespace kudu
-#endif /* KUDU_INTEGRATION_TESTS_EXTERNAL_MINI_CLUSTER_H */
http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc b/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc
index 3690f47..c93182b 100644
--- a/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc
+++ b/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc
@@ -79,8 +79,8 @@ int ExternalMiniClusterFsInspector::CountFilesInDir(const string& path,
}
int ExternalMiniClusterFsInspector::CountWALFilesOnTS(int index) {
- string data_dir = cluster_->tablet_server(index)->data_dir();
- string ts_wal_dir = JoinPathSegments(data_dir, FsManager::kWalDirName);
+ string ts_wal_dir = JoinPathSegments(cluster_->tablet_server(index)->wal_dir(),
+ FsManager::kWalDirName);
vector<string> tablets;
CHECK_OK(ListFilesInDir(ts_wal_dir, &tablets));
int total_segments = 0;
@@ -109,8 +109,8 @@ vector<string> ExternalMiniClusterFsInspector::ListTabletsOnTS(int index) {
}
vector<string> ExternalMiniClusterFsInspector::ListTabletsWithDataOnTS(int index) {
- string data_dir = cluster_->tablet_server(index)->data_dir();
- string wal_dir = JoinPathSegments(data_dir, FsManager::kWalDirName);
+ string wal_dir = JoinPathSegments(cluster_->tablet_server(index)->wal_dir(),
+ FsManager::kWalDirName);
vector<string> tablets;
CHECK_OK(ListFilesInDir(wal_dir, &tablets));
return tablets;
@@ -120,8 +120,8 @@ int ExternalMiniClusterFsInspector::CountFilesInWALDirForTS(
int index,
const string& tablet_id,
StringPiece pattern) {
- string data_dir = cluster_->tablet_server(index)->data_dir();
- string wal_dir = JoinPathSegments(data_dir, FsManager::kWalDirName);
+ string wal_dir = JoinPathSegments(cluster_->tablet_server(index)->wal_dir(),
+ FsManager::kWalDirName);
string tablet_wal_dir = JoinPathSegments(wal_dir, tablet_id);
if (!env_->FileExists(tablet_wal_dir)) {
return 0;
@@ -336,8 +336,8 @@ Status ExternalMiniClusterFsInspector::WaitForFilePatternInTabletWalDirOnTs(
Status s;
MonoTime deadline = MonoTime::Now() + timeout;
- string data_dir = cluster_->tablet_server(ts_index)->data_dir();
- string ts_wal_dir = JoinPathSegments(data_dir, FsManager::kWalDirName);
+ string ts_wal_dir = JoinPathSegments(cluster_->tablet_server(ts_index)->wal_dir(),
+ FsManager::kWalDirName);
string tablet_wal_dir = JoinPathSegments(ts_wal_dir, tablet_id);
string error_msg;
http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/log_verifier.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/log_verifier.cc b/src/kudu/integration-tests/log_verifier.cc
index 9a8f919..05fefd1 100644
--- a/src/kudu/integration-tests/log_verifier.cc
+++ b/src/kudu/integration-tests/log_verifier.cc
@@ -60,13 +60,13 @@ LogVerifier::~LogVerifier() {
Status LogVerifier::OpenFsManager(ExternalTabletServer* ets,
unique_ptr<FsManager>* fs) {
- const string& data_dir = ets->data_dir();
FsManagerOpts fs_opts;
fs_opts.read_only = true;
- fs_opts.wal_path = data_dir;
+ fs_opts.wal_path = ets->wal_dir();
+ fs_opts.data_paths = ets->data_dirs();
unique_ptr<FsManager> ret(new FsManager(Env::Default(), fs_opts));
RETURN_NOT_OK_PREPEND(ret->Open(),
- Substitute("Couldn't initialize FS Manager for $0", data_dir));
+ Substitute("Couldn't initialize FS Manager for $0", ets->wal_dir()));
fs->swap(ret);
return Status::OK();
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/master_failover-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/master_failover-itest.cc b/src/kudu/integration-tests/master_failover-itest.cc
index 5288e89..ed96b7f 100644
--- a/src/kudu/integration-tests/master_failover-itest.cc
+++ b/src/kudu/integration-tests/master_failover-itest.cc
@@ -363,8 +363,7 @@ TEST_F(MasterFailoverTest, TestMasterPermanentFailure) {
// "Fail" a master and blow away its state completely.
failed_master->Shutdown();
- string data_root = failed_master->data_dir();
- env_->DeleteRecursively(data_root);
+ ASSERT_OK(failed_master->DeleteFromDisk());
// Pick another master at random to serve as a basis for recovery.
//
@@ -382,7 +381,7 @@ TEST_F(MasterFailoverTest, TestMasterPermanentFailure) {
"local_replica",
"cmeta",
"print_replica_uuids",
- "--fs_wal_dir=" + other_master->data_dir(),
+ "--fs_wal_dir=" + other_master->wal_dir(),
"--fs_data_dirs=" + other_master->data_dir(),
master::SysCatalogTable::kSysCatalogTabletId
};
@@ -408,8 +407,8 @@ TEST_F(MasterFailoverTest, TestMasterPermanentFailure) {
kBinPath,
"fs",
"format",
- "--fs_wal_dir=" + data_root,
- "--fs_data_dirs=" + data_root,
+ "--fs_wal_dir=" + failed_master->wal_dir(),
+ "--fs_data_dirs=" + failed_master->data_dir(),
"--uuid=" + uuid
};
ASSERT_OK(Subprocess::Call(args));
@@ -421,8 +420,8 @@ TEST_F(MasterFailoverTest, TestMasterPermanentFailure) {
kBinPath,
"local_replica",
"copy_from_remote",
- "--fs_wal_dir=" + data_root,
- "--fs_data_dirs=" + data_root,
+ "--fs_wal_dir=" + failed_master->wal_dir(),
+ "--fs_data_dirs=" + failed_master->data_dir(),
master::SysCatalogTable::kSysCatalogTabletId,
other_master->bound_rpc_hostport().ToString()
};
http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/master_migration-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/master_migration-itest.cc b/src/kudu/integration-tests/master_migration-itest.cc
index d3fed08..29f2b2d 100644
--- a/src/kudu/integration-tests/master_migration-itest.cc
+++ b/src/kudu/integration-tests/master_migration-itest.cc
@@ -110,13 +110,15 @@ TEST_F(MasterMigrationTest, TestEndToEndMigration) {
// Format a filesystem tree for each of the new masters and get the uuids.
for (int i = 1; i < kMasterRpcPorts.size(); i++) {
string data_root = cluster_->GetDataPath(Substitute("master-$0", i));
+ string wal_dir = cluster_->GetWalPath(Substitute("master-$0", i));
ASSERT_OK(env_->CreateDir(DirName(data_root)));
+ ASSERT_OK(env_->CreateDir(wal_dir));
{
vector<string> args = {
kBinPath,
"fs",
"format",
- "--fs_wal_dir=" + data_root,
+ "--fs_wal_dir=" + wal_dir,
"--fs_data_dirs=" + data_root
};
ASSERT_OK(Subprocess::Call(args));
@@ -127,7 +129,7 @@ TEST_F(MasterMigrationTest, TestEndToEndMigration) {
"fs",
"dump",
"uuid",
- "--fs_wal_dir=" + data_root,
+ "--fs_wal_dir=" + wal_dir,
"--fs_data_dirs=" + data_root
};
string uuid;
@@ -145,7 +147,7 @@ TEST_F(MasterMigrationTest, TestEndToEndMigration) {
"local_replica",
"cmeta",
"rewrite_raft_config",
- "--fs_wal_dir=" + data_root,
+ "--fs_wal_dir=" + cluster_->GetWalPath("master-0"),
"--fs_data_dirs=" + data_root,
SysCatalogTable::kSysCatalogTabletId
};
@@ -168,11 +170,12 @@ TEST_F(MasterMigrationTest, TestEndToEndMigration) {
// filesystems.
for (int i = 1; i < kMasterRpcPorts.size(); i++) {
string data_root = cluster_->GetDataPath(Substitute("master-$0", i));
+ string wal_dir = cluster_->GetWalPath(Substitute("master-$0", i));
vector<string> args = {
kBinPath,
"local_replica",
"copy_from_remote",
- "--fs_wal_dir=" + data_root,
+ "--fs_wal_dir=" + wal_dir,
"--fs_data_dirs=" + data_root,
SysCatalogTable::kSysCatalogTabletId,
cluster_->master()->bound_rpc_hostport().ToString()
http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/multidir_cluster-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/multidir_cluster-itest.cc b/src/kudu/integration-tests/multidir_cluster-itest.cc
new file mode 100644
index 0000000..5b04ef4
--- /dev/null
+++ b/src/kudu/integration-tests/multidir_cluster-itest.cc
@@ -0,0 +1,92 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include "kudu/client/client.h"
+#include "kudu/gutil/map-util.h"
+#include "kudu/integration-tests/external_mini_cluster-itest-base.h"
+#include "kudu/integration-tests/test_workload.h"
+#include "kudu/util/path_util.h"
+
+namespace kudu {
+
+using std::map;
+using std::string;
+using std::vector;
+
+class MultiDirClusterITest : public ExternalMiniClusterITestBase {};
+
+TEST_F(MultiDirClusterITest, TestBasicMultiDirCluster) {
+ const uint32_t kNumDataDirs = 3;
+ vector<string> ts_flags = {
+ // Flush frequently to trigger writes.
+ "--flush_threshold_mb=1",
+ "--flush_threshold_secs=1",
+
+ // Spread tablet data across all data dirs.
+ "--fs_target_data_dirs_per_tablet=0"
+ };
+
+ NO_FATALS(StartCluster(ts_flags, {}, /* num_tablet_servers= */ 1, kNumDataDirs));
+ ExternalTabletServer* ts = cluster_->tablet_server(0);
+ TestWorkload work(cluster_.get());
+ work.set_num_replicas(1);
+ work.Setup();
+
+ // Check that all daemons have the expected number of directories.
+ ASSERT_EQ(kNumDataDirs, cluster_->master()->data_dirs().size());
+ ASSERT_EQ(kNumDataDirs, ts->data_dirs().size());
+
+ // Take an initial snapshot of the number of files in each directory.
+ map<string, int> num_files_in_each_dir;
+ for (const string& data_dir : ts->data_dirs()) {
+ string data_path = JoinPathSegments(data_dir, "data");
+ vector<string> files;
+ ASSERT_OK(inspect_->ListFilesInDir(data_path, &files));
+ InsertOrDie(&num_files_in_each_dir, data_dir, files.size());
+ }
+
+ work.Start();
+ ASSERT_EVENTUALLY([&] {
+ // Check that files are being written to more than one directory.
+ int num_dirs_added_to = 0;
+ for (const string& data_dir : ts->data_dirs()) {
+ string data_path = JoinPathSegments(data_dir, "data");
+ vector<string> files;
+ inspect_->ListFilesInDir(data_path, &files);
+ int* num_files_before_insert = FindOrNull(num_files_in_each_dir, data_dir);
+ ASSERT_NE(nullptr, num_files_before_insert);
+ if (*num_files_before_insert < files.size()) {
+ num_dirs_added_to++;
+ }
+ }
+ // Block placement should guarantee that more than one data dir will have
+ // data written to it.
+ ASSERT_GT(num_dirs_added_to, 1);
+ vector<string> wal_files;
+ ASSERT_OK(inspect_->ListFilesInDir(JoinPathSegments(ts->wal_dir(), "wals"), &wal_files));
+ ASSERT_FALSE(wal_files.empty());
+ });
+ work.StopAndJoin();
+}
+
+} // namespace kudu
http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/open-readonly-fs-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/open-readonly-fs-itest.cc b/src/kudu/integration-tests/open-readonly-fs-itest.cc
index 68e7ba0..e17902d 100644
--- a/src/kudu/integration-tests/open-readonly-fs-itest.cc
+++ b/src/kudu/integration-tests/open-readonly-fs-itest.cc
@@ -114,8 +114,8 @@ TEST_F(OpenReadonlyFsITest, TestWriteAndVerify) {
auto t = std::thread([this, deadline] () {
FsManagerOpts fs_opts;
fs_opts.read_only = true;
- fs_opts.wal_path = cluster_->tablet_server(0)->data_dir();
- fs_opts.data_paths = { cluster_->tablet_server(0)->data_dir() };
+ fs_opts.wal_path = cluster_->tablet_server(0)->wal_dir();
+ fs_opts.data_paths = cluster_->tablet_server(0)->data_dirs();
while (MonoTime::Now() < deadline) {
FsManager fs(Env::Default(), fs_opts);
CHECK_OK(fs.Open());
http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/ts_itest-base.h
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/ts_itest-base.h b/src/kudu/integration-tests/ts_itest-base.h
index 898a349..7c96bdd 100644
--- a/src/kudu/integration-tests/ts_itest-base.h
+++ b/src/kudu/integration-tests/ts_itest-base.h
@@ -86,7 +86,8 @@ class TabletServerIntegrationTestBase : public TabletServerTestBase {
void CreateCluster(const std::string& data_root_path,
const std::vector<std::string>& non_default_ts_flags,
- const std::vector<std::string>& non_default_master_flags) {
+ const std::vector<std::string>& non_default_master_flags,
+ uint32_t num_data_dirs = 1) {
LOG(INFO) << "Starting cluster with:";
LOG(INFO) << "--------------";
@@ -97,6 +98,7 @@ class TabletServerIntegrationTestBase : public TabletServerTestBase {
ExternalMiniClusterOptions opts;
opts.num_tablet_servers = FLAGS_num_tablet_servers;
opts.data_root = GetTestPath(data_root_path);
+ opts.num_data_dirs = num_data_dirs;
// Enable exactly once semantics for tests.
http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/ts_recovery-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/ts_recovery-itest.cc b/src/kudu/integration-tests/ts_recovery-itest.cc
index a145cd6..e996ad1 100644
--- a/src/kudu/integration-tests/ts_recovery-itest.cc
+++ b/src/kudu/integration-tests/ts_recovery-itest.cc
@@ -322,7 +322,10 @@ TEST_F(TsRecoveryITestDeathTest, TestRecoverFromOpIdOverflow) {
{
// Append a no-op to the WAL with an overflowed term and index to simulate a
// crash after KUDU-1933.
- gscoped_ptr<FsManager> fs_manager(new FsManager(env_, ets->data_dir()));
+ FsManagerOpts opts;
+ opts.wal_path = ets->wal_dir();
+ opts.data_paths = ets->data_dirs();
+ gscoped_ptr<FsManager> fs_manager(new FsManager(env_, opts));
ASSERT_OK(fs_manager->Open());
scoped_refptr<Clock> clock(new HybridClock());
ASSERT_OK(clock->Init());