You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by mp...@apache.org on 2017/01/19 19:09:44 UTC
[1/2] kudu git commit: docs: add notes for multi master migration
Repository: kudu
Updated Branches:
refs/heads/master 7d16929c4 -> 8598bd947
docs: add notes for multi master migration
Change-Id: I7e469bc23b650f3b99c785311acc665ac20f33ea
Reviewed-on: http://gerrit.cloudera.org:8080/5728
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/7679f9b7
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/7679f9b7
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/7679f9b7
Branch: refs/heads/master
Commit: 7679f9b7ef70982885c8141aa8136d4a2a237305
Parents: 7d16929
Author: zhangzhen <zh...@xiaomi.com>
Authored: Wed Jan 18 11:27:23 2017 +0800
Committer: Adar Dembo <ad...@cloudera.com>
Committed: Thu Jan 19 10:54:24 2017 +0000
----------------------------------------------------------------------
docs/administration.adoc | 23 +++++++++++++++++------
1 file changed, 17 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/7679f9b7/docs/administration.adoc
----------------------------------------------------------------------
diff --git a/docs/administration.adoc b/docs/administration.adoc
index 1911dc4..aab5b94 100644
--- a/docs/administration.adoc
+++ b/docs/administration.adoc
@@ -227,8 +227,10 @@ WARNING: All of the command line steps below should be executed as the Kudu UNIX
. Perform the following preparatory steps for the existing master:
* Identify and record the directory where the master's data lives. If using Kudu system packages,
- the default value is /var/lib/kudu/master, but it may be customized via the `fs_wal_dir`
- configuration parameter.
+ the default value is /var/lib/kudu/master, but it may be customized via the `fs_wal_dir` and
+ `fs_data_dirs` configuration parameter. Please note if you've set fs_data_dirs to some directories
+ other than the value of fs_wal_dir, it should be explicitly included in every command below where
+ fs_wal_dir is also included.
* Identify and record the port the master is using for RPCs. The default port value is 7051, but it
may have been customized using the `rpc_bind_addresses` configuration parameter.
* Identify the master's UUID. It can be fetched using the following command:
@@ -317,6 +319,11 @@ Example::
$ kudu local_replica cmeta rewrite_raft_config --fs_wal_dir=/var/lib/kudu/master 00000000000000000000000000000000 4aab798a69e94fab8d77069edff28ce0:master-1:7051 f5624e05f40649b79a757629a69d061e:master-2:7051 988d8ac6530f426cbe180be5ba52033d:master-3:7051
----
+. Modify the value of the `master_addresses` configuration parameter for both existing master and new masters.
+ The new value must be a comma-separated list of all of the masters. Each entry is a string of the form `<hostname>:<port>`
+hostname:: master's previously recorded hostname or alias
+port:: master's previously recorded RPC port number
+
. Start the existing master.
. Copy the master data to each new master with the following command, executed on each new master
@@ -404,8 +411,10 @@ WARNING: All of the command line steps below should be executed as the Kudu UNIX
. Perform the following preparatory steps for each live master:
* Identify and record the directory where the master's data lives. If using Kudu system packages,
- the default value is /var/lib/kudu/master, but it may be customized via the `fs_wal_dir`
- configuration parameter.
+ the default value is /var/lib/kudu/master, but it may be customized via the `fs_wal_dir` and
+ `fs_data_dirs` configuration parameter. Please note if you've set fs_data_dirs to some directories
+ other than the value of fs_wal_dir, it should be explicitly included in every command below where
+ fs_wal_dir is also included.
* Identify and record the master's UUID. It can be fetched using the following command:
+
[source,bash]
@@ -424,8 +433,10 @@ $ kudu fs dump uuid --fs_wal_dir=/var/lib/kudu/master 2>/dev/null
+
. Perform the following preparatory steps for the reference master:
* Identify and record the directory where the master's data lives. If using Kudu system packages,
- the default value is /var/lib/kudu/master, but it may be customized via the `fs_wal_dir`
- configuration parameter.
+ the default value is /var/lib/kudu/master, but it may be customized via the `fs_wal_dir` and
+ `fs_data_dirs` configuration parameter. Please note if you've set fs_data_dirs to some directories
+ other than the value of fs_wal_dir, it should be explicitly included in every command below where
+ fs_wal_dir is also included.
* Identify and record the UUIDs of every master in the cluster, using the following command:
+
[source,bash]
[2/2] kudu git commit: Add some path / env related helper functions
Posted by mp...@apache.org.
Add some path / env related helper functions
* path_util: Add SplitPath() helper function
This function provides a (theoretically) portable method of splitting
a filesystem path into its constituent components.
* env: Add GetCurrentWorkingDir() and ChangeDir()
These are functions that were missing from our Env implementation.
* env_util: Implement CreateDirsRecursively()
This function is similar in spirit to the "mkdir -p" command.
Also reordered a few includes and deleted a few lines of commented-out code.
Change-Id: Ia664708a09493923abbf2ff4e56e3d49c62cf97e
Reviewed-on: http://gerrit.cloudera.org:8080/5618
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Mike Percy <mp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/8598bd94
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/8598bd94
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/8598bd94
Branch: refs/heads/master
Commit: 8598bd94779aa9da966b5b4cb5e613db46a60b5a
Parents: 7679f9b
Author: Mike Percy <mp...@apache.org>
Authored: Thu Jan 5 15:54:01 2017 -0800
Committer: Mike Percy <mp...@apache.org>
Committed: Thu Jan 19 19:09:15 2017 +0000
----------------------------------------------------------------------
src/kudu/util/env-test.cc | 18 ++++++++++
src/kudu/util/env.h | 6 ++++
src/kudu/util/env_posix.cc | 21 +++++++++++
src/kudu/util/env_util-test.cc | 69 ++++++++++++++++++++++++++++++++----
src/kudu/util/env_util.cc | 32 ++++++++++++++---
src/kudu/util/env_util.h | 6 ++++
src/kudu/util/path_util-test.cc | 16 +++++++++
src/kudu/util/path_util.cc | 15 ++++++++
src/kudu/util/path_util.h | 4 +++
9 files changed, 176 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/8598bd94/src/kudu/util/env-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/env-test.cc b/src/kudu/util/env-test.cc
index a90f0e1..01e0db6 100644
--- a/src/kudu/util/env-test.cc
+++ b/src/kudu/util/env-test.cc
@@ -826,4 +826,22 @@ TEST_F(TestEnv, TestGetBytesFree) {
<< "Failed after " << kIters << " attempts";
}
+TEST_F(TestEnv, TestChangeDir) {
+ string orig_dir;
+ ASSERT_OK(env_->GetCurrentWorkingDir(&orig_dir));
+
+ string cwd;
+ ASSERT_OK(env_->ChangeDir("/"));
+ ASSERT_OK(env_->GetCurrentWorkingDir(&cwd));
+ ASSERT_EQ("/", cwd);
+
+ ASSERT_OK(env_->ChangeDir(test_dir_));
+ ASSERT_OK(env_->GetCurrentWorkingDir(&cwd));
+ ASSERT_EQ(test_dir_, cwd);
+
+ ASSERT_OK(env_->ChangeDir(orig_dir));
+ ASSERT_OK(env_->GetCurrentWorkingDir(&cwd));
+ ASSERT_EQ(orig_dir, cwd);
+}
+
} // namespace kudu
http://git-wip-us.apache.org/repos/asf/kudu/blob/8598bd94/src/kudu/util/env.h
----------------------------------------------------------------------
diff --git a/src/kudu/util/env.h b/src/kudu/util/env.h
index e585b85..78d4bb5 100644
--- a/src/kudu/util/env.h
+++ b/src/kudu/util/env.h
@@ -152,6 +152,12 @@ class Env {
// Delete the specified directory.
virtual Status DeleteDir(const std::string& dirname) = 0;
+ // Return the current working directory.
+ virtual Status GetCurrentWorkingDir(std::string* cwd) const = 0;
+
+ // Change the current working directory.
+ virtual Status ChangeDir(const std::string& dest) = 0;
+
// Synchronize the entry for a specific directory.
virtual Status SyncDir(const std::string& dirname) = 0;
http://git-wip-us.apache.org/repos/asf/kudu/blob/8598bd94/src/kudu/util/env_posix.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/env_posix.cc b/src/kudu/util/env_posix.cc
index 2516f61..2a83fa8 100644
--- a/src/kudu/util/env_posix.cc
+++ b/src/kudu/util/env_posix.cc
@@ -890,6 +890,27 @@ class PosixEnv : public Env {
return result;
};
+ Status GetCurrentWorkingDir(string* cwd) const override {
+ TRACE_EVENT0("io", "PosixEnv::GetCurrentWorkingDir");
+ ThreadRestrictions::AssertIOAllowed();
+ unique_ptr<char, FreeDeleter> wd(getcwd(NULL, 0));
+ if (!wd) {
+ return IOError("getcwd()", errno);
+ }
+ cwd->assign(wd.get());
+ return Status::OK();
+ }
+
+ Status ChangeDir(const string& dest) override {
+ TRACE_EVENT1("io", "PosixEnv::ChangeDir", "dest", dest);
+ ThreadRestrictions::AssertIOAllowed();
+ Status result;
+ if (chdir(dest.c_str()) != 0) {
+ result = IOError(dest, errno);
+ }
+ return result;
+ }
+
virtual Status SyncDir(const std::string& dirname) OVERRIDE {
TRACE_EVENT1("io", "SyncDir", "path", dirname);
ThreadRestrictions::AssertIOAllowed();
http://git-wip-us.apache.org/repos/asf/kudu/blob/8598bd94/src/kudu/util/env_util-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/env_util-test.cc b/src/kudu/util/env_util-test.cc
index 3676d64..90b308d 100644
--- a/src/kudu/util/env_util-test.cc
+++ b/src/kudu/util/env_util-test.cc
@@ -15,41 +15,98 @@
// specific language governing permissions and limitations
// under the License.
+#include <unistd.h>
+
#include "kudu/util/env_util.h"
#include <gflags/gflags.h>
#include <memory>
#include <sys/statvfs.h>
+#include "kudu/gutil/strings/substitute.h"
+#include "kudu/util/path_util.h"
+#include "kudu/util/test_macros.h"
#include "kudu/util/test_util.h"
DECLARE_int64(disk_reserved_bytes);
DECLARE_int64(disk_reserved_bytes_free_for_testing);
-namespace kudu {
-
using std::string;
using std::unique_ptr;
+using strings::Substitute;
+
+namespace kudu {
+namespace env_util {
class EnvUtilTest: public KuduTest {
};
TEST_F(EnvUtilTest, TestDiskSpaceCheck) {
- Env* env = Env::Default();
-
const int64_t kRequestedBytes = 0;
int64_t reserved_bytes = 0;
- ASSERT_OK(env_util::VerifySufficientDiskSpace(env, test_dir_, kRequestedBytes, reserved_bytes));
+ ASSERT_OK(VerifySufficientDiskSpace(env_, test_dir_, kRequestedBytes, reserved_bytes));
// Make it seem as if the disk is full and specify that we should have
// reserved 200 bytes. Even asking for 0 bytes should return an error
// indicating we are out of space.
FLAGS_disk_reserved_bytes_free_for_testing = 0;
reserved_bytes = 200;
- Status s = env_util::VerifySufficientDiskSpace(env, test_dir_, kRequestedBytes, reserved_bytes);
+ Status s = VerifySufficientDiskSpace(env_, test_dir_, kRequestedBytes, reserved_bytes);
ASSERT_TRUE(s.IsIOError());
ASSERT_EQ(ENOSPC, s.posix_code());
ASSERT_STR_CONTAINS(s.ToString(), "Insufficient disk space");
}
+// Ensure that we can recursively create directories using both absolute and
+// relative paths.
+TEST_F(EnvUtilTest, TestCreateDirsRecursively) {
+ // Absolute path.
+ string path = JoinPathSegments(test_dir_, "a/b/c");
+ ASSERT_OK(CreateDirsRecursively(env_, path));
+ bool is_dir;
+ ASSERT_OK(env_->IsDirectory(path, &is_dir));
+ ASSERT_TRUE(is_dir);
+
+ // Repeating the previous command should also succeed (it should be a no-op).
+ ASSERT_OK(CreateDirsRecursively(env_, path));
+ ASSERT_OK(env_->IsDirectory(path, &is_dir));
+ ASSERT_TRUE(is_dir);
+
+ // Relative path.
+ ASSERT_OK(env_->ChangeDir(test_dir_)); // Change to test dir to keep CWD clean.
+ string rel_base = Substitute("$0-$1", CURRENT_TEST_CASE_NAME(), CURRENT_TEST_NAME());
+ ASSERT_FALSE(env_->FileExists(rel_base));
+ path = JoinPathSegments(rel_base, "x/y/z");
+ ASSERT_OK(CreateDirsRecursively(env_, path));
+ ASSERT_OK(env_->IsDirectory(path, &is_dir));
+ ASSERT_TRUE(is_dir);
+
+ // Directory creation should fail if a file is a part of the path.
+ path = JoinPathSegments(test_dir_, "x/y/z");
+ string file_path = JoinPathSegments(test_dir_, "x"); // Conflicts with 'path'.
+ ASSERT_FALSE(env_->FileExists(path));
+ ASSERT_FALSE(env_->FileExists(file_path));
+ // Create an empty file in the path.
+ unique_ptr<WritableFile> out;
+ ASSERT_OK(env_->NewWritableFile(file_path, &out));
+ ASSERT_OK(out->Close());
+ ASSERT_TRUE(env_->FileExists(file_path));
+ // Fail.
+ Status s = CreateDirsRecursively(env_, path);
+ ASSERT_TRUE(s.IsAlreadyPresent()) << s.ToString();
+ ASSERT_STR_CONTAINS(s.ToString(), "File exists");
+
+ // We should be able to create a directory tree even when a symlink exists as
+ // part of the path.
+ path = JoinPathSegments(test_dir_, "link/a/b");
+ string link_path = JoinPathSegments(test_dir_, "link");
+ string real_dir = JoinPathSegments(test_dir_, "real_dir");
+ ASSERT_OK(env_->CreateDir(real_dir));
+ PCHECK(symlink(real_dir.c_str(), link_path.c_str()) == 0);
+ ASSERT_OK(CreateDirsRecursively(env_, path));
+ ASSERT_OK(env_->IsDirectory(path, &is_dir));
+ ASSERT_TRUE(is_dir);
+}
+
+} // namespace env_util
} // namespace kudu
http://git-wip-us.apache.org/repos/asf/kudu/blob/8598bd94/src/kudu/util/env_util.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/env_util.cc b/src/kudu/util/env_util.cc
index 27cdbba..e398831 100644
--- a/src/kudu/util/env_util.cc
+++ b/src/kudu/util/env_util.cc
@@ -15,10 +15,13 @@
// specific language governing permissions and limitations
// under the License.
+#include "kudu/util/env_util.h"
+
#include <algorithm>
#include <memory>
#include <string>
#include <utility>
+#include <vector>
#include <gflags/gflags.h>
#include <glog/logging.h>
@@ -30,9 +33,9 @@
#include "kudu/gutil/strings/util.h"
#include "kudu/util/debug-util.h"
#include "kudu/util/env.h"
-#include "kudu/util/env_util.h"
-#include "kudu/util/status.h"
#include "kudu/util/flag_tags.h"
+#include "kudu/util/path_util.h"
+#include "kudu/util/status.h"
DEFINE_int64(disk_reserved_bytes_free_for_testing, -1,
"For testing only! Set to number of bytes free on each filesystem. "
@@ -56,9 +59,6 @@ DEFINE_string(disk_reserved_override_prefix_2_path_for_testing, "",
DEFINE_int64(disk_reserved_override_prefix_2_bytes_free_for_testing, -1,
"For testing only! Set number of bytes free on the path prefix specified by "
"--disk_reserved_override_prefix_2_path_for_testing. Set to -1 to disable.");
-//DEFINE_string(disk_reserved_prefixes_with_bytes_free_for_testing, "",
-// "For testing only! Syntax: '/path/a:5,/path/b:7' means a has 5 bytes free, "
-// "b has 7 bytes free. Set to empty string to disable this test-specific override.");
TAG_FLAG(disk_reserved_override_prefix_1_path_for_testing, unsafe);
TAG_FLAG(disk_reserved_override_prefix_2_path_for_testing, unsafe);
TAG_FLAG(disk_reserved_override_prefix_1_bytes_free_for_testing, unsafe);
@@ -69,6 +69,7 @@ TAG_FLAG(disk_reserved_override_prefix_2_bytes_free_for_testing, runtime);
using std::shared_ptr;
using std::string;
using std::unique_ptr;
+using std::vector;
using strings::Substitute;
namespace kudu {
@@ -189,6 +190,27 @@ Status CreateDirIfMissing(Env* env, const string& path, bool* created) {
return s.IsAlreadyPresent() ? Status::OK() : s;
}
+Status CreateDirsRecursively(Env* env, const string& path) {
+ vector<string> segments = SplitPath(path);
+ string partial_path;
+ for (const string& segment : segments) {
+ partial_path = partial_path.empty() ? segment : JoinPathSegments(partial_path, segment);
+ bool is_dir;
+ Status s = env->IsDirectory(partial_path, &is_dir);
+ if (s.ok()) {
+ // We didn't get a NotFound error, so something is there.
+ if (is_dir) continue; // It's a normal directory.
+ // Maybe a file or a symlink. Let's try to follow the symlink.
+ string real_partial_path;
+ RETURN_NOT_OK(env->Canonicalize(partial_path, &real_partial_path));
+ s = env->IsDirectory(real_partial_path, &is_dir);
+ if (s.ok() && is_dir) continue; // It's a symlink to a directory.
+ }
+ RETURN_NOT_OK_PREPEND(env->CreateDir(partial_path), "Unable to create directory");
+ }
+ return Status::OK();
+}
+
Status CopyFile(Env* env, const string& source_path, const string& dest_path,
WritableFileOptions opts) {
unique_ptr<SequentialFile> source;
http://git-wip-us.apache.org/repos/asf/kudu/blob/8598bd94/src/kudu/util/env_util.h
----------------------------------------------------------------------
diff --git a/src/kudu/util/env_util.h b/src/kudu/util/env_util.h
index f3f40ae..c54bfa8 100644
--- a/src/kudu/util/env_util.h
+++ b/src/kudu/util/env_util.h
@@ -68,6 +68,12 @@ Status ReadFully(RandomAccessFile* file, uint64_t offset, size_t n,
Status CreateDirIfMissing(Env* env, const std::string& path,
bool* created = NULL);
+// Recursively create directories, if they do not exist, along the given path.
+// Returns OK if successful or if the given path already existed.
+// Upon failure, it is possible that some part of the directory structure may
+// have been successfully created. Emulates the behavior of `mkdir -p`.
+Status CreateDirsRecursively(Env* env, const std::string& path);
+
// Copy the contents of file source_path to file dest_path.
// This is not atomic, and if there is an error while reading or writing,
// a partial copy may be left in 'dest_path'. Does not fsync the parent
http://git-wip-us.apache.org/repos/asf/kudu/blob/8598bd94/src/kudu/util/path_util-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/path_util-test.cc b/src/kudu/util/path_util-test.cc
index d36eba6..0d617fc 100644
--- a/src/kudu/util/path_util-test.cc
+++ b/src/kudu/util/path_util-test.cc
@@ -15,10 +15,16 @@
// specific language governing permissions and limitations
// under the License.
+#include <string>
+#include <vector>
+
#include <gtest/gtest.h>
#include "kudu/util/path_util.h"
+using std::string;
+using std::vector;
+
namespace kudu {
TEST(TestPathUtil, BaseNameTest) {
@@ -58,4 +64,14 @@ TEST(TestPathUtil, DirNameTest) {
ASSERT_EQ("/ab", DirName("/ab/cd"));
}
+TEST(TestPathUtil, SplitPathTest) {
+ typedef vector<string> Vec;
+ ASSERT_EQ(Vec({"/"}), SplitPath("/"));
+ ASSERT_EQ(Vec({"/", "a", "b"}), SplitPath("/a/b"));
+ ASSERT_EQ(Vec({"/", "a", "b"}), SplitPath("/a/b/"));
+ ASSERT_EQ(Vec({"a", "b"}), SplitPath("a/b"));
+ ASSERT_EQ(Vec({"."}), SplitPath("."));
+ ASSERT_EQ(Vec(), SplitPath(""));
+}
+
} // namespace kudu
http://git-wip-us.apache.org/repos/asf/kudu/blob/8598bd94/src/kudu/util/path_util.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/path_util.cc b/src/kudu/util/path_util.cc
index 0cbf890..6a095cc 100644
--- a/src/kudu/util/path_util.cc
+++ b/src/kudu/util/path_util.cc
@@ -24,12 +24,16 @@
#include <string>
#include "kudu/gutil/gscoped_ptr.h"
+#include "kudu/gutil/strings/split.h"
#if defined(__APPLE__)
#include <mutex>
#endif // defined(__APPLE__)
using std::string;
+using std::vector;
+using strings::SkipEmpty;
+using strings::Split;
namespace kudu {
@@ -49,6 +53,17 @@ std::string JoinPathSegments(const std::string &a,
}
}
+vector<string> SplitPath(const string& path) {
+ if (path.empty()) return {};
+ vector<string> segments;
+ if (path[0] == '/') segments.push_back("/");
+ vector<StringPiece> pieces = Split(path, "/", SkipEmpty());
+ for (const StringPiece& piece : pieces) {
+ segments.emplace_back(piece.data(), piece.size());
+ }
+ return segments;
+}
+
string DirName(const string& path) {
gscoped_ptr<char[], FreeDeleter> path_copy(strdup(path.c_str()));
#if defined(__APPLE__)
http://git-wip-us.apache.org/repos/asf/kudu/blob/8598bd94/src/kudu/util/path_util.h
----------------------------------------------------------------------
diff --git a/src/kudu/util/path_util.h b/src/kudu/util/path_util.h
index 4b5d082..bb5c631 100644
--- a/src/kudu/util/path_util.h
+++ b/src/kudu/util/path_util.h
@@ -20,6 +20,7 @@
#define KUDU_UTIL_PATH_UTIL_H
#include <string>
+#include <vector>
namespace kudu {
@@ -33,6 +34,9 @@ extern const char kOldTmpInfix[];
std::string JoinPathSegments(const std::string &a,
const std::string &b);
+// Split a path into segments with the appropriate path separator.
+std::vector<std::string> SplitPath(const std::string& path);
+
// Return the enclosing directory of path.
// This is like dirname(3) but for C++ strings.
std::string DirName(const std::string& path);