You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by ad...@apache.org on 2016/08/16 01:38:19 UTC
[2/3] kudu git commit: fs: allow format with user-specified uuid
fs: allow format with user-specified uuid
The CLI tool is going to use this for the "handling permanent failure"
workflow.
Change-Id: Ib4189d6150a263b7dde304dd19a449d3c0ab6c8c
Reviewed-on: http://gerrit.cloudera.org:8080/3968
Reviewed-by: Todd Lipcon <to...@apache.org>
Tested-by: Adar Dembo <ad...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/1bce7312
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/1bce7312
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/1bce7312
Branch: refs/heads/master
Commit: 1bce73126799cb046755c0555bbe5be7622c9ace
Parents: aa94185
Author: Adar Dembo <ad...@cloudera.com>
Authored: Thu Aug 11 21:14:36 2016 -0700
Committer: Adar Dembo <ad...@cloudera.com>
Committed: Tue Aug 16 01:32:58 2016 +0000
----------------------------------------------------------------------
src/kudu/fs/fs_manager-test.cc | 25 ++++++++++++++--
src/kudu/fs/fs_manager.cc | 17 ++++++++---
src/kudu/fs/fs_manager.h | 11 +++++--
src/kudu/util/CMakeLists.txt | 1 +
src/kudu/util/oid_generator-test.cc | 50 ++++++++++++++++++++++++++++++++
src/kudu/util/oid_generator.cc | 37 +++++++++++++++++++----
src/kudu/util/oid_generator.h | 12 ++++++++
7 files changed, 139 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/1bce7312/src/kudu/fs/fs_manager-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/fs_manager-test.cc b/src/kudu/fs/fs_manager-test.cc
index 922e617..21bb460 100644
--- a/src/kudu/fs/fs_manager-test.cc
+++ b/src/kudu/fs/fs_manager-test.cc
@@ -21,12 +21,15 @@
#include "kudu/fs/block_manager.h"
#include "kudu/fs/fs_manager.h"
+#include "kudu/gutil/strings/substitute.h"
#include "kudu/gutil/strings/util.h"
#include "kudu/util/metrics.h"
+#include "kudu/util/oid_generator.h"
#include "kudu/util/test_macros.h"
#include "kudu/util/test_util.h"
using std::shared_ptr;
+using strings::Substitute;
namespace kudu {
@@ -155,7 +158,7 @@ TEST_F(FsManagerTestBase, TestCannotUseNonEmptyFsRoot) {
}
TEST_F(FsManagerTestBase, TestEmptyWALPath) {
- ReinitFsManager("", vector<string>());
+ ReinitFsManager("", {});
Status s = fs_manager()->CreateInitialFileSystemLayout();
ASSERT_TRUE(s.IsIOError());
ASSERT_STR_CONTAINS(s.ToString(), "directory (fs_wal_dir) not provided");
@@ -165,7 +168,7 @@ TEST_F(FsManagerTestBase, TestOnlyWALPath) {
string path = GetTestPath("new_fs_root");
ASSERT_OK(env_->CreateDir(path));
- ReinitFsManager(path, vector<string>());
+ ReinitFsManager(path, {});
ASSERT_OK(fs_manager()->CreateInitialFileSystemLayout());
ASSERT_TRUE(HasPrefixString(fs_manager()->GetWalsRootDir(), path));
ASSERT_TRUE(HasPrefixString(fs_manager()->GetConsensusMetadataDir(), path));
@@ -175,4 +178,22 @@ TEST_F(FsManagerTestBase, TestOnlyWALPath) {
ASSERT_TRUE(HasPrefixString(data_dirs[0], path));
}
+TEST_F(FsManagerTestBase, TestFormatWithSpecificUUID) {
+ string path = GetTestPath("new_fs_root");
+ ReinitFsManager(path, {});
+
+ // Use an invalid uuid at first.
+ string uuid = "not_a_valid_uuid";
+ Status s = fs_manager()->CreateInitialFileSystemLayout(uuid);
+ ASSERT_TRUE(s.IsInvalidArgument());
+ ASSERT_STR_CONTAINS(s.ToString(), Substitute("invalid uuid $0", uuid));
+
+ // Now use a valid one.
+ ObjectIdGenerator oid_generator;
+ uuid = oid_generator.Next();
+ ASSERT_OK(fs_manager()->CreateInitialFileSystemLayout(uuid));
+ ASSERT_OK(fs_manager()->Open());
+ ASSERT_EQ(uuid, fs_manager()->uuid());
+}
+
} // namespace kudu
http://git-wip-us.apache.org/repos/asf/kudu/blob/1bce7312/src/kudu/fs/fs_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/fs_manager.cc b/src/kudu/fs/fs_manager.cc
index 117389f..86e5d4f 100644
--- a/src/kudu/fs/fs_manager.cc
+++ b/src/kudu/fs/fs_manager.cc
@@ -22,6 +22,7 @@
#include <map>
#include <unordered_set>
+#include <boost/optional.hpp>
#include <glog/logging.h>
#include <glog/stl_logging.h>
#include <google/protobuf/message.h>
@@ -244,7 +245,7 @@ Status FsManager::Open() {
return Status::OK();
}
-Status FsManager::CreateInitialFileSystemLayout() {
+Status FsManager::CreateInitialFileSystemLayout(boost::optional<string> uuid) {
CHECK(!read_only_);
RETURN_NOT_OK(Init());
@@ -271,7 +272,7 @@ Status FsManager::CreateInitialFileSystemLayout() {
ElementDeleter d(&delete_on_failure);
InstanceMetadataPB metadata;
- CreateInstanceMetadata(&metadata);
+ RETURN_NOT_OK(CreateInstanceMetadata(std::move(uuid), &metadata));
unordered_set<string> to_sync;
for (const string& root : canonicalized_all_fs_roots_) {
bool created;
@@ -319,9 +320,16 @@ Status FsManager::CreateInitialFileSystemLayout() {
return Status::OK();
}
-void FsManager::CreateInstanceMetadata(InstanceMetadataPB* metadata) {
+Status FsManager::CreateInstanceMetadata(boost::optional<string> uuid,
+ InstanceMetadataPB* metadata) {
ObjectIdGenerator oid_generator;
- metadata->set_uuid(oid_generator.Next());
+ if (uuid) {
+ string canonicalized_uuid;
+ RETURN_NOT_OK(oid_generator.Canonicalize(uuid.get(), &canonicalized_uuid));
+ metadata->set_uuid(canonicalized_uuid);
+ } else {
+ metadata->set_uuid(oid_generator.Next());
+ }
string time_str;
StringAppendStrftime(&time_str, "%Y-%m-%d %H:%M:%S", time(nullptr), false);
@@ -330,6 +338,7 @@ void FsManager::CreateInstanceMetadata(InstanceMetadataPB* metadata) {
hostname = "<unknown host>";
}
metadata->set_format_stamp(Substitute("Formatted at $0 on $1", time_str, hostname));
+ return Status::OK();
}
Status FsManager::WriteInstanceMetadata(const InstanceMetadataPB& metadata,
http://git-wip-us.apache.org/repos/asf/kudu/blob/1bce7312/src/kudu/fs/fs_manager.h
----------------------------------------------------------------------
diff --git a/src/kudu/fs/fs_manager.h b/src/kudu/fs/fs_manager.h
index fab0436..3c4420a 100644
--- a/src/kudu/fs/fs_manager.h
+++ b/src/kudu/fs/fs_manager.h
@@ -18,6 +18,8 @@
#ifndef KUDU_FS_FS_MANAGER_H
#define KUDU_FS_FS_MANAGER_H
+#include <boost/none.hpp>
+#include <boost/optional/optional.hpp>
#include <gtest/gtest_prod.h>
#include <iosfwd>
#include <memory>
@@ -107,10 +109,12 @@ class FsManager {
// the on-disk structures.
Status Open();
- // Create the initial filesystem layout.
+ // Create the initial filesystem layout. If 'uuid' is provided, uses it as
+ // uuid of the filesystem. Otherwise generates one at random.
//
// Returns an error if the file system is already initialized.
- Status CreateInitialFileSystemLayout();
+ Status CreateInitialFileSystemLayout(
+ boost::optional<std::string> uuid = boost::none);
void DumpFileSystemTree(std::ostream& out);
@@ -212,7 +216,8 @@ class FsManager {
void InitBlockManager();
// Create a new InstanceMetadataPB.
- void CreateInstanceMetadata(InstanceMetadataPB* metadata);
+ Status CreateInstanceMetadata(boost::optional<std::string> uuid,
+ InstanceMetadataPB* metadata);
// Save a InstanceMetadataPB to the filesystem.
// Does not mutate the current state of the fsmanager.
http://git-wip-us.apache.org/repos/asf/kudu/blob/1bce7312/src/kudu/util/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/kudu/util/CMakeLists.txt b/src/kudu/util/CMakeLists.txt
index 4a000b6..a4527b9 100644
--- a/src/kudu/util/CMakeLists.txt
+++ b/src/kudu/util/CMakeLists.txt
@@ -309,6 +309,7 @@ ADD_KUDU_TEST(mt-threadlocal-test RUN_SERIAL true)
ADD_KUDU_TEST(net/dns_resolver-test)
ADD_KUDU_TEST(net/net_util-test)
ADD_KUDU_TEST(object_pool-test)
+ADD_KUDU_TEST(oid_generator-test)
ADD_KUDU_TEST(once-test)
ADD_KUDU_TEST(os-util-test)
ADD_KUDU_TEST(path_util-test)
http://git-wip-us.apache.org/repos/asf/kudu/blob/1bce7312/src/kudu/util/oid_generator-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/oid_generator-test.cc b/src/kudu/util/oid_generator-test.cc
new file mode 100644
index 0000000..a38b496
--- /dev/null
+++ b/src/kudu/util/oid_generator-test.cc
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "kudu/util/oid_generator.h"
+
+#include <gtest/gtest.h>
+#include <string>
+
+#include "kudu/util/test_util.h"
+
+using std::string;
+
+namespace kudu {
+
+TEST(ObjectIdGeneratorTest, TestCanoicalizeUuid) {
+ ObjectIdGenerator gen;
+ const string kExpectedCanonicalized = "0123456789abcdef0123456789abcdef";
+ string canonicalized;
+ Status s = gen.Canonicalize("not_a_uuid", &canonicalized);
+ {
+ SCOPED_TRACE(s.ToString());
+ ASSERT_TRUE(s.IsInvalidArgument());
+ ASSERT_STR_CONTAINS(s.ToString(), "invalid uuid");
+ }
+ ASSERT_OK(gen.Canonicalize(
+ "01234567-89ab-cdef-0123-456789abcdef", &canonicalized));
+ ASSERT_EQ(kExpectedCanonicalized, canonicalized);
+ ASSERT_OK(gen.Canonicalize(
+ "0123456789abcdef0123456789abcdef", &canonicalized));
+ ASSERT_EQ(kExpectedCanonicalized, canonicalized);
+ ASSERT_OK(gen.Canonicalize(
+ "0123456789AbCdEf0123456789aBcDeF", &canonicalized));
+ ASSERT_EQ(kExpectedCanonicalized, canonicalized);
+}
+
+} // namespace kudu
http://git-wip-us.apache.org/repos/asf/kudu/blob/1bce7312/src/kudu/util/oid_generator.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/oid_generator.cc b/src/kudu/util/oid_generator.cc
index da7acd8..580463c 100644
--- a/src/kudu/util/oid_generator.cc
+++ b/src/kudu/util/oid_generator.cc
@@ -15,21 +15,48 @@
// specific language governing permissions and limitations
// under the License.
+#include "kudu/util/oid_generator.h"
+
+#include <boost/uuid/uuid_generators.hpp>
+#include <exception>
#include <mutex>
#include <string>
#include "kudu/gutil/stringprintf.h"
-#include "kudu/util/oid_generator.h"
+#include "kudu/gutil/strings/substitute.h"
+#include "kudu/util/status.h"
+
+using strings::Substitute;
namespace kudu {
-string ObjectIdGenerator::Next() {
- std::lock_guard<LockType> l(oid_lock_);
- boost::uuids::uuid oid = oid_generator_();
- const uint8_t *uuid = oid.data;
+namespace {
+
+string ConvertUuidToString(const boost::uuids::uuid& to_convert) {
+ const uint8_t* uuid = to_convert.data;
return StringPrintf("%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x",
uuid[0], uuid[1], uuid[2], uuid[3], uuid[4], uuid[5], uuid[6], uuid[7],
uuid[8], uuid[9], uuid[10], uuid[11], uuid[12], uuid[13], uuid[14], uuid[15]);
}
+} // anonymous namespace
+
+string ObjectIdGenerator::Next() {
+ std::lock_guard<LockType> l(oid_lock_);
+ boost::uuids::uuid uuid = oid_generator_();
+ return ConvertUuidToString(uuid);
+}
+
+Status ObjectIdGenerator::Canonicalize(const string& input,
+ string* output) const {
+ try {
+ boost::uuids::uuid uuid = oid_validator_(input);
+ *output = ConvertUuidToString(uuid);
+ return Status::OK();
+ } catch (std::exception& e) {
+ return Status::InvalidArgument(Substitute("invalid uuid $0: $1",
+ input, e.what()));
+ }
+}
+
} // namespace kudu
http://git-wip-us.apache.org/repos/asf/kudu/blob/1bce7312/src/kudu/util/oid_generator.h
----------------------------------------------------------------------
diff --git a/src/kudu/util/oid_generator.h b/src/kudu/util/oid_generator.h
index 85a7412..7acccc9 100644
--- a/src/kudu/util/oid_generator.h
+++ b/src/kudu/util/oid_generator.h
@@ -23,6 +23,7 @@
#include "kudu/gutil/macros.h"
#include "kudu/util/locks.h"
+#include "kudu/util/status.h"
namespace kudu {
@@ -33,15 +34,26 @@ class ObjectIdGenerator {
ObjectIdGenerator() {}
~ObjectIdGenerator() {}
+ // Generates and returns a new UUID.
std::string Next();
+ // Validates an existing UUID and converts it into the format used by Kudu
+ // (that is, 16 hexadecimal bytes without any dashes).
+ Status Canonicalize(const std::string& input, std::string* output) const;
+
private:
DISALLOW_COPY_AND_ASSIGN(ObjectIdGenerator);
typedef simple_spinlock LockType;
+ // Protects 'oid_generator_'.
LockType oid_lock_;
+
+ // Generates new UUIDs.
boost::uuids::random_generator oid_generator_;
+
+ // Validates provided UUIDs.
+ boost::uuids::string_generator oid_validator_;
};
} // namespace kudu