You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by to...@apache.org on 2017/10/18 19:25:34 UTC
[5/5] kudu git commit: fs: change uuid index from uint16_t to int
fs: change uuid index from uint16_t to int
While the file block manager is restricted to 64k data directories (because
the data dir's uuid index is included in the upper 16 bits of the FBM's
block IDs), there's no such restriction for the log block manager. As such,
these types should all be int; the FBM can rely on an existing runtime check
in DataDirManager::Open to enforce that no more than 64k directories exist.
Note: I found all of these by grepping for uint16_t and excluding ports and
data types.
Change-Id: I81c8c0588b7bc8ac4a506592f8ac4c77d00fbd8b
Reviewed-on: http://gerrit.cloudera.org:8080/8287
Tested-by: Kudu Jenkins
Reviewed-by: Todd Lipcon <to...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/f96ab36a
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/f96ab36a
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/f96ab36a
Branch: refs/heads/master
Commit: f96ab36adea6d653641cd09fdb9ed92ee8d0c8bc
Parents: 892a5fc
Author: Adar Dembo <ad...@cloudera.com>
Authored: Thu Oct 12 13:50:47 2017 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Wed Oct 18 19:23:55 2017 +0000
----------------------------------------------------------------------
src/kudu/fs/data_dirs-test.cc | 16 ++++-----
src/kudu/fs/data_dirs.cc | 56 +++++++++++++++---------------
src/kudu/fs/data_dirs.h | 52 +++++++++++++--------------
src/kudu/fs/file_block_manager.cc | 19 +++++-----
src/kudu/fs/fs_manager.cc | 2 +-
src/kudu/fs/log_block_manager-test.cc | 4 +--
src/kudu/fs/log_block_manager.cc | 6 ++--
src/kudu/tserver/ts_tablet_manager.cc | 2 +-
8 files changed, 80 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/f96ab36a/src/kudu/fs/data_dirs-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/data_dirs-test.cc b/src/kudu/fs/data_dirs-test.cc
index c6dd144..0c29046 100644
--- a/src/kudu/fs/data_dirs-test.cc
+++ b/src/kudu/fs/data_dirs-test.cc
@@ -201,7 +201,7 @@ TEST_F(DataDirsTest, TestFailedDirNotReturned) {
ASSERT_OK(dd_manager_->CreateDataDirGroup(test_tablet_name_));
DataDir* dd;
DataDir* failed_dd;
- uint16_t uuid_idx;
+ int uuid_idx;
// Fail one of the directories in the group and verify that it is not used.
ASSERT_OK(dd_manager_->GetNextDataDir(test_block_opts_, &failed_dd));
ASSERT_TRUE(dd_manager_->FindUuidIndexByDataDir(failed_dd, &uuid_idx));
@@ -241,13 +241,13 @@ TEST_F(DataDirsTest, TestFailedDirNotAddedToGroup) {
// Check that all uuid_indices are valid and are not in the failed directory
// (uuid_idx 0).
for (const string& uuid : pb.uuids()) {
- uint16_t* uuid_idx = FindOrNull(dd_manager_->idx_by_uuid_, uuid);
+ int* uuid_idx = FindOrNull(dd_manager_->idx_by_uuid_, uuid);
ASSERT_NE(nullptr, uuid_idx);
ASSERT_NE(0, *uuid_idx);
}
dd_manager_->DeleteDataDirGroup(test_tablet_name_);
- for (uint16_t i = 1; i < kNumDirs - 1; i++) {
+ for (int i = 1; i < kNumDirs - 1; i++) {
ASSERT_OK(dd_manager_->MarkDataDirFailed(i));
}
Status s = dd_manager_->MarkDataDirFailed(kNumDirs - 1);
@@ -310,9 +310,9 @@ TEST_F(DataDirsTest, TestLoadBalancingBias) {
// Note: this should not happen in the wild and is used here as a way to
// introduce some initial skew to the distribution.
auto uuid_idx_iter = dd_manager_->tablets_by_uuid_idx_map_.begin();
- vector<uint16_t> skewed_dir_indices;
+ vector<int> skewed_dir_indices;
for (int i = 0; i < kNumSkewedDirs; i++) {
- uint16_t uuid_idx = uuid_idx_iter->first;
+ int uuid_idx = uuid_idx_iter->first;
skewed_dir_indices.push_back(uuid_idx);
uuid_idx_iter++;
}
@@ -321,7 +321,7 @@ TEST_F(DataDirsTest, TestLoadBalancingBias) {
for (int skew_tablet_idx = 0; skew_tablet_idx < kTabletsPerSkewedDir; skew_tablet_idx++) {
string skew_tablet = Substitute("$0-$1", kSkewTabletPrefix, skew_tablet_idx);
InsertOrDie(&dd_manager_->group_by_tablet_map_, skew_tablet, DataDirGroup(skewed_dir_indices));
- for (uint16_t uuid_idx : skewed_dir_indices) {
+ for (int uuid_idx : skewed_dir_indices) {
InsertOrDie(&FindOrDie(dd_manager_->tablets_by_uuid_idx_map_, uuid_idx), skew_tablet);
}
}
@@ -352,7 +352,7 @@ TEST_F(DataDirsTest, TestLoadBalancingBias) {
// having the mean, 10, tablets. Instead, the block-placement heuristic should
// not completely ignore the initially skewed dirs.
bool some_added_to_skewed_dirs = false;
- for (uint16_t skewed_uuid_index : skewed_dir_indices) {
+ for (int skewed_uuid_index : skewed_dir_indices) {
set<string>* tablets = FindOrNull(dd_manager_->tablets_by_uuid_idx_map_, skewed_uuid_index);
ASSERT_NE(nullptr, tablets);
if (tablets->size() > kTabletsPerSkewedDir) {
@@ -401,7 +401,7 @@ TEST_F(DataDirManagerTest, TestOpenWithFailedDirs) {
// The directory manager will successfully open with the single failed directory.
ASSERT_OK(OpenDataDirManager());
- set<uint16_t> failed_dirs;
+ set<int> failed_dirs;
ASSERT_EQ(1, dd_manager_->GetFailedDataDirs().size());
// Now fail almost all of the other directories, leaving the first intact.
http://git-wip-us.apache.org/repos/asf/kudu/blob/f96ab36a/src/kudu/fs/data_dirs.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/data_dirs.cc b/src/kudu/fs/data_dirs.cc
index ac839b0..1143514 100644
--- a/src/kudu/fs/data_dirs.cc
+++ b/src/kudu/fs/data_dirs.cc
@@ -431,7 +431,7 @@ Status DataDirManager::Open() {
} else {
lock_mode = LockMode::MANDATORY;
}
- int max_data_dirs = block_manager_type_ == "file" ? (1 << 16) - 1 : kuint32max;
+ const int kMaxDataDirs = block_manager_type_ == "file" ? (1 << 16) - 1 : kint32max;
int i = 0;
// Create a directory for all data dirs specified by the user.
@@ -529,7 +529,7 @@ Status DataDirManager::Open() {
TabletsByUuidIndexMap tablets_by_uuid_idx_map;
FailedDataDirSet failed_data_dirs;
- const auto insert_to_maps = [&] (uint16_t idx, string uuid, DataDir* dd) {
+ const auto insert_to_maps = [&] (int idx, string uuid, DataDir* dd) {
InsertOrDie(&uuid_by_root, DirName(dd->dir()), uuid);
InsertOrDie(&uuid_by_idx, idx, uuid);
InsertOrDie(&idx_by_uuid, uuid, idx);
@@ -547,7 +547,7 @@ Status DataDirManager::Open() {
continue;
}
const PathSetPB& path_set = dd->instance()->metadata()->path_set();
- uint32_t idx = -1;
+ int idx = -1;
for (int i = 0; i < path_set.all_uuids_size(); i++) {
if (path_set.uuid() == path_set.all_uuids(i)) {
idx = i;
@@ -555,9 +555,9 @@ Status DataDirManager::Open() {
}
}
DCHECK_NE(idx, -1); // Guaranteed by CheckIntegrity().
- if (idx > max_data_dirs) {
+ if (idx > kMaxDataDirs) {
return Status::NotSupported(
- Substitute("Block manager supports a maximum of $0 paths", max_data_dirs));
+ Substitute("Block manager supports a maximum of $0 paths", kMaxDataDirs));
}
insert_to_maps(idx, path_set.uuid(), dd.get());
}
@@ -597,7 +597,7 @@ Status DataDirManager::Open() {
// Initialize the 'fullness' status of the data directories.
for (const auto& dd : data_dirs_) {
- uint16_t uuid_idx;
+ int uuid_idx;
CHECK(FindUuidIndexByDataDir(dd.get(), &uuid_idx));
if (ContainsKey(failed_data_dirs_, uuid_idx)) {
continue;
@@ -626,7 +626,7 @@ Status DataDirManager::LoadDataDirGroupFromPB(const std::string& tablet_id,
return Status::AlreadyPresent("Tried to load directory group for tablet but one is already "
"registered", tablet_id);
}
- for (uint16_t uuid_idx : group_from_pb.uuid_indices()) {
+ for (int uuid_idx : group_from_pb.uuid_indices()) {
InsertOrDie(&FindOrDie(tablets_by_uuid_idx_map_, uuid_idx), tablet_id);
}
return Status::OK();
@@ -647,7 +647,7 @@ Status DataDirManager::CreateDataDirGroup(const string& tablet_id,
group_target_size = std::min(FLAGS_fs_target_data_dirs_per_tablet,
static_cast<int>(data_dirs_.size()));
}
- vector<uint16_t> group_indices;
+ vector<int> group_indices;
if (mode == DirDistributionMode::ACROSS_ALL_DIRS) {
// If using all dirs, add all regardless of directory state.
AppendKeysFromMap(data_dir_by_uuid_idx_, &group_indices);
@@ -675,7 +675,7 @@ Status DataDirManager::CreateDataDirGroup(const string& tablet_id,
}
}
InsertOrDie(&group_by_tablet_map_, tablet_id, DataDirGroup(group_indices));
- for (uint16_t uuid_idx : group_indices) {
+ for (int uuid_idx : group_indices) {
InsertOrDie(&FindOrDie(tablets_by_uuid_idx_map_, uuid_idx), tablet_id);
}
return Status::OK();
@@ -683,8 +683,8 @@ Status DataDirManager::CreateDataDirGroup(const string& tablet_id,
Status DataDirManager::GetNextDataDir(const CreateBlockOptions& opts, DataDir** dir) {
shared_lock<rw_spinlock> lock(dir_group_lock_.get_lock());
- const vector<uint16_t>* group_uuid_indices;
- vector<uint16_t> valid_uuid_indices;
+ const vector<int>* group_uuid_indices;
+ vector<int> valid_uuid_indices;
if (PREDICT_TRUE(!opts.tablet_id.empty())) {
// Get the data dir group for the tablet.
DataDirGroup* group = FindOrNull(group_by_tablet_map_, opts.tablet_id);
@@ -715,7 +715,7 @@ Status DataDirManager::GetNextDataDir(const CreateBlockOptions& opts, DataDir**
// Randomly select a member of the group that is not full.
for (int i : random_indices) {
- uint16_t uuid_idx = (*group_uuid_indices)[i];
+ int uuid_idx = (*group_uuid_indices)[i];
DataDir* candidate = FindOrDie(data_dir_by_uuid_idx_, uuid_idx);
RETURN_NOT_OK(candidate->RefreshIsFull(DataDir::RefreshMode::EXPIRED_ONLY));
if (!candidate->is_full()) {
@@ -744,7 +744,7 @@ void DataDirManager::DeleteDataDirGroup(const std::string& tablet_id) {
return;
}
// Remove the tablet_id from every data dir in its group.
- for (uint16_t uuid_idx : group->uuid_indices()) {
+ for (int uuid_idx : group->uuid_indices()) {
FindOrDie(tablets_by_uuid_idx_map_, uuid_idx).erase(tablet_id);
}
group_by_tablet_map_.erase(tablet_id);
@@ -762,9 +762,9 @@ bool DataDirManager::GetDataDirGroupPB(const std::string& tablet_id,
}
Status DataDirManager::GetDirsForGroupUnlocked(int target_size,
- vector<uint16_t>* group_indices) {
+ vector<int>* group_indices) {
DCHECK(dir_group_lock_.is_locked());
- vector<uint16_t> candidate_indices;
+ vector<int> candidate_indices;
for (auto& e : data_dir_by_uuid_idx_) {
if (ContainsKey(failed_data_dirs_, e.first)) {
continue;
@@ -792,16 +792,16 @@ Status DataDirManager::GetDirsForGroupUnlocked(int target_size,
return Status::OK();
}
-DataDir* DataDirManager::FindDataDirByUuidIndex(uint16_t uuid_idx) const {
+DataDir* DataDirManager::FindDataDirByUuidIndex(int uuid_idx) const {
DCHECK_LT(uuid_idx, data_dirs_.size());
return FindPtrOrNull(data_dir_by_uuid_idx_, uuid_idx);
}
-bool DataDirManager::FindUuidIndexByDataDir(DataDir* dir, uint16_t* uuid_idx) const {
+bool DataDirManager::FindUuidIndexByDataDir(DataDir* dir, int* uuid_idx) const {
return FindCopy(uuid_idx_by_data_dir_, dir, uuid_idx);
}
-bool DataDirManager::FindUuidIndexByRoot(const string& root, uint16_t* uuid_idx) const {
+bool DataDirManager::FindUuidIndexByRoot(const string& root, int* uuid_idx) const {
string uuid;
if (FindUuidByRoot(root, &uuid)) {
return FindUuidIndexByUuid(uuid, uuid_idx);
@@ -809,7 +809,7 @@ bool DataDirManager::FindUuidIndexByRoot(const string& root, uint16_t* uuid_idx)
return false;
}
-bool DataDirManager::FindUuidIndexByUuid(const string& uuid, uint16_t* uuid_idx) const {
+bool DataDirManager::FindUuidIndexByUuid(const string& uuid, int* uuid_idx) const {
return FindCopy(idx_by_uuid_, uuid, uuid_idx);
}
@@ -817,7 +817,7 @@ bool DataDirManager::FindUuidByRoot(const string& root, string* uuid) const {
return FindCopy(uuid_by_root_, root, uuid);
}
-set<string> DataDirManager::FindTabletsByDataDirUuidIdx(uint16_t uuid_idx) const {
+set<string> DataDirManager::FindTabletsByDataDirUuidIdx(int uuid_idx) const {
DCHECK_LT(uuid_idx, data_dirs_.size());
shared_lock<rw_spinlock> lock(dir_group_lock_.get_lock());
const set<string>* tablet_set_ptr = FindOrNull(tablets_by_uuid_idx_map_, uuid_idx);
@@ -828,12 +828,12 @@ set<string> DataDirManager::FindTabletsByDataDirUuidIdx(uint16_t uuid_idx) const
}
void DataDirManager::MarkDataDirFailedByUuid(const string& uuid) {
- uint16_t uuid_idx;
+ int uuid_idx;
CHECK(FindUuidIndexByUuid(uuid, &uuid_idx));
WARN_NOT_OK(MarkDataDirFailed(uuid_idx), "Failed to handle disk failure");
}
-Status DataDirManager::MarkDataDirFailed(uint16_t uuid_idx, const string& error_message) {
+Status DataDirManager::MarkDataDirFailed(int uuid_idx, const string& error_message) {
DCHECK_LT(uuid_idx, data_dirs_.size());
std::lock_guard<percpu_rwlock> lock(dir_group_lock_);
DataDir* dd = FindDataDirByUuidIndex(uuid_idx);
@@ -856,15 +856,15 @@ Status DataDirManager::MarkDataDirFailed(uint16_t uuid_idx, const string& error_
return Status::OK();
}
-bool DataDirManager::IsDataDirFailed(uint16_t uuid_idx) const {
+bool DataDirManager::IsDataDirFailed(int uuid_idx) const {
DCHECK_LT(uuid_idx, data_dirs_.size());
shared_lock<rw_spinlock> lock(dir_group_lock_.get_lock());
return ContainsKey(failed_data_dirs_, uuid_idx);
}
bool DataDirManager::IsTabletInFailedDir(const string& tablet_id) const {
- const set<uint16_t> failed_dirs = GetFailedDataDirs();
- for (uint16_t failed_dir : failed_dirs) {
+ const set<int> failed_dirs = GetFailedDataDirs();
+ for (int failed_dir : failed_dirs) {
if (ContainsKey(FindTabletsByDataDirUuidIdx(failed_dir), tablet_id)) {
return true;
}
@@ -872,13 +872,13 @@ bool DataDirManager::IsTabletInFailedDir(const string& tablet_id) const {
return false;
}
-void DataDirManager::RemoveUnhealthyDataDirsUnlocked(const vector<uint16_t>& uuid_indices,
- vector<uint16_t>* healthy_indices) const {
+void DataDirManager::RemoveUnhealthyDataDirsUnlocked(const vector<int>& uuid_indices,
+ vector<int>* healthy_indices) const {
if (PREDICT_TRUE(failed_data_dirs_.empty())) {
return;
}
healthy_indices->clear();
- for (uint16_t uuid_idx : uuid_indices) {
+ for (int uuid_idx : uuid_indices) {
DCHECK_LT(uuid_idx, data_dirs_.size());
if (!ContainsKey(failed_data_dirs_, uuid_idx)) {
healthy_indices->emplace_back(uuid_idx);
http://git-wip-us.apache.org/repos/asf/kudu/blob/f96ab36a/src/kudu/fs/data_dirs.h
----------------------------------------------------------------------
diff --git a/src/kudu/fs/data_dirs.h b/src/kudu/fs/data_dirs.h
index 0ac01da..1dca128 100644
--- a/src/kudu/fs/data_dirs.h
+++ b/src/kudu/fs/data_dirs.h
@@ -57,8 +57,8 @@ typedef std::vector<CanonicalizedRootAndStatus> CanonicalizedRootsList;
namespace fs {
-typedef std::unordered_map<uint16_t, std::string> UuidByUuidIndexMap;
-typedef std::unordered_map<std::string, uint16_t> UuidIndexByUuidMap;
+typedef std::unordered_map<int, std::string> UuidByUuidIndexMap;
+typedef std::unordered_map<std::string, int> UuidIndexByUuidMap;
class PathInstanceMetadataFile;
struct CreateBlockOptions;
@@ -68,20 +68,20 @@ const char kInstanceMetadataFileName[] = "block_manager_instance";
namespace internal {
// A DataDirGroup is a group of directories used by an entity for block
-// placement. A group is represented in memory by a list of 2-byte indices,
-// which index into the list of all UUIDs found in a PathSetPB. A group is
-// represented on-disk as a list of full UUIDs, and as such, when writing or
-// reading from disk, a mapping is needed to translate between index and UUID.
+// placement. A group is represented in memory by a list of indices which index
+// into the list of all UUIDs found in a PathSetPB. A group is represented
+// on-disk as a list of full UUIDs, and as such, when writing or reading from
+// disk, a mapping is needed to translate between index and UUID.
//
// The same directory may appear in multiple DataDirGroups.
class DataDirGroup {
public:
- explicit DataDirGroup(std::vector<uint16_t> uuid_indices)
+ explicit DataDirGroup(std::vector<int> uuid_indices)
: uuid_indices_(std::move(uuid_indices)) {}
static DataDirGroup FromPB(const DataDirGroupPB& pb,
const UuidIndexByUuidMap& uuid_idx_by_uuid) {
- std::vector<uint16_t> uuid_indices;
+ std::vector<int> uuid_indices;
for (const std::string& uuid : pb.uuids()) {
uuid_indices.push_back(FindOrDie(uuid_idx_by_uuid, uuid));
}
@@ -92,17 +92,17 @@ class DataDirGroup {
DataDirGroupPB* pb) const {
DCHECK(pb);
DataDirGroupPB group;
- for (uint16_t uuid_idx : uuid_indices_) {
+ for (int uuid_idx : uuid_indices_) {
group.add_uuids(FindOrDie(uuid_by_uuid_idx, uuid_idx));
}
pb->Swap(&group);
}
- const std::vector<uint16_t>& uuid_indices() const { return uuid_indices_; }
+ const std::vector<int>& uuid_indices() const { return uuid_indices_; }
private:
// UUID indices corresponding to the data directories within the group.
- const std::vector<uint16_t> uuid_indices_;
+ const std::vector<int> uuid_indices_;
};
} // namespace internal
@@ -313,7 +313,7 @@ class DataDirManager {
// Finds the set of tablet_ids in the data dir specified by 'uuid_idx' and
// returns a copy, returning an empty set if none are found.
- std::set<std::string> FindTabletsByDataDirUuidIdx(uint16_t uuid_idx) const;
+ std::set<std::string> FindTabletsByDataDirUuidIdx(int uuid_idx) const;
// ==========================================================================
// Directory Health
@@ -324,19 +324,19 @@ class DataDirManager {
// describing what directories are affected.
//
// Returns an error if all directories have failed.
- Status MarkDataDirFailed(uint16_t uuid_idx, const std::string& error_message = "");
+ Status MarkDataDirFailed(int uuid_idx, const std::string& error_message = "");
// Fails the directory specified by 'uuid' and logs a warning if all
// directories have failed.
void MarkDataDirFailedByUuid(const std::string& uuid);
// Returns whether or not the 'uuid_idx' refers to a failed directory.
- bool IsDataDirFailed(uint16_t uuid_idx) const;
+ bool IsDataDirFailed(int uuid_idx) const;
// Returns whether the tablet's data is spread across a failed directory.
bool IsTabletInFailedDir(const std::string& tablet_id) const;
- const std::set<uint16_t> GetFailedDataDirs() const {
+ const std::set<int> GetFailedDataDirs() const {
shared_lock<rw_spinlock> group_lock(dir_group_lock_.get_lock());
return failed_data_dirs_;
}
@@ -360,16 +360,16 @@ class DataDirManager {
//
// More information on uuid indexes and their relation to data directories
// can be found next to PathSetPB in fs.proto.
- DataDir* FindDataDirByUuidIndex(uint16_t uuid_idx) const;
+ DataDir* FindDataDirByUuidIndex(int uuid_idx) const;
// Finds a uuid index by data directory, returning false if it can't be found.
- bool FindUuidIndexByDataDir(DataDir* dir, uint16_t* uuid_idx) const;
+ bool FindUuidIndexByDataDir(DataDir* dir, int* uuid_idx) const;
// Finds a uuid index by root path, returning false if it can't be found.
- bool FindUuidIndexByRoot(const std::string& root, uint16_t* uuid_idx) const;
+ bool FindUuidIndexByRoot(const std::string& root, int* uuid_idx) const;
// Finds a uuid index by UUID, returning false if it can't be found.
- bool FindUuidIndexByUuid(const std::string& uuid, uint16_t* uuid_idx) const;
+ bool FindUuidIndexByUuid(const std::string& uuid, int* uuid_idx) const;
// Finds a UUID by canonicalized root name, returning false if it can't be found.
bool FindUuidByRoot(const std::string& root, std::string* uuid) const;
@@ -414,12 +414,12 @@ class DataDirManager {
// added. Although this function does not itself change DataDirManager state,
// its expected usage warrants that it is called within the scope of a
// lock_guard of dir_group_lock_.
- Status GetDirsForGroupUnlocked(int target_size, std::vector<uint16_t>* group_indices);
+ Status GetDirsForGroupUnlocked(int target_size, std::vector<int>* group_indices);
// Goes through the data dirs in 'uuid_indices' and populates
// 'healthy_indices' with those that haven't failed.
- void RemoveUnhealthyDataDirsUnlocked(const std::vector<uint16_t>& uuid_indices,
- std::vector<uint16_t>* healthy_indices) const;
+ void RemoveUnhealthyDataDirsUnlocked(const std::vector<int>& uuid_indices,
+ std::vector<int>* healthy_indices) const;
Env* env_;
const std::string block_manager_type_;
@@ -438,22 +438,22 @@ class DataDirManager {
typedef std::unordered_map<std::string, std::string> UuidByRootMap;
UuidByRootMap uuid_by_root_;
- typedef std::unordered_map<uint16_t, DataDir*> UuidIndexMap;
+ typedef std::unordered_map<int, DataDir*> UuidIndexMap;
UuidIndexMap data_dir_by_uuid_idx_;
- typedef std::unordered_map<DataDir*, uint16_t> ReverseUuidIndexMap;
+ typedef std::unordered_map<DataDir*, int> ReverseUuidIndexMap;
ReverseUuidIndexMap uuid_idx_by_data_dir_;
typedef std::unordered_map<std::string, internal::DataDirGroup> TabletDataDirGroupMap;
TabletDataDirGroupMap group_by_tablet_map_;
- typedef std::unordered_map<uint16_t, std::set<std::string>> TabletsByUuidIndexMap;
+ typedef std::unordered_map<int, std::set<std::string>> TabletsByUuidIndexMap;
TabletsByUuidIndexMap tablets_by_uuid_idx_map_;
UuidByUuidIndexMap uuid_by_idx_;
UuidIndexByUuidMap idx_by_uuid_;
- typedef std::set<uint16_t> FailedDataDirSet;
+ typedef std::set<int> FailedDataDirSet;
FailedDataDirSet failed_data_dirs_;
// Lock protecting access to the dir group maps and to failed_data_dirs_.
http://git-wip-us.apache.org/repos/asf/kudu/blob/f96ab36a/src/kudu/fs/file_block_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/file_block_manager.cc b/src/kudu/fs/file_block_manager.cc
index 6694b88..788c191 100644
--- a/src/kudu/fs/file_block_manager.cc
+++ b/src/kudu/fs/file_block_manager.cc
@@ -37,6 +37,7 @@
#include "kudu/fs/fs_report.h"
#include "kudu/gutil/bind.h"
#include "kudu/gutil/casts.h"
+#include "kudu/gutil/integral_types.h"
#include "kudu/gutil/map-util.h"
#include "kudu/gutil/port.h"
#include "kudu/gutil/ref_counted.h"
@@ -99,7 +100,7 @@ class FileBlockLocation {
// Construct a location from its constituent parts.
static FileBlockLocation FromParts(DataDir* data_dir,
- uint16_t data_dir_idx,
+ int data_dir_idx,
const BlockId& block_id);
// Construct a location from a full block ID.
@@ -107,7 +108,7 @@ class FileBlockLocation {
const BlockId& block_id);
// Get the data dir index of a given block ID.
- static uint16_t GetDataDirIdx(const BlockId& block_id) {
+ static int GetDataDirIdx(const BlockId& block_id) {
return block_id.id() >> 48;
}
@@ -154,8 +155,10 @@ class FileBlockLocation {
};
FileBlockLocation FileBlockLocation::FromParts(DataDir* data_dir,
- uint16_t data_dir_idx,
+ int data_dir_idx,
const BlockId& block_id) {
+ DCHECK_LT(data_dir_idx, kuint16max);
+
// The combined ID consists of 'data_dir_idx' (top 2 bytes) and 'block_id'
// (bottom 6 bytes). The top 2 bytes of 'block_id' are dropped.
uint64_t combined_id = static_cast<uint64_t>(data_dir_idx) << 48;
@@ -704,14 +707,14 @@ Status FileBlockManager::Open(FsReport* report) {
// Prepare the filesystem report and either return or log it.
FsReport local_report;
- set<uint16_t> failed_dirs = dd_manager_->GetFailedDataDirs();
+ set<int> failed_dirs = dd_manager_->GetFailedDataDirs();
for (const auto& dd : dd_manager_->data_dirs()) {
// Don't report failed directories.
// TODO(KUDU-2111): currently the FsReport only reports on containers for
// the log block manager. Implement some sort of reporting for failed
// directories as well.
if (PREDICT_FALSE(!failed_dirs.empty())) {
- uint16_t uuid_idx;
+ int uuid_idx;
CHECK(dd_manager_->FindUuidIndexByDataDir(dd.get(), &uuid_idx));
if (ContainsKey(failed_dirs, uuid_idx)) {
continue;
@@ -734,7 +737,7 @@ Status FileBlockManager::CreateBlock(const CreateBlockOptions& opts,
DataDir* dir;
RETURN_NOT_OK(dd_manager_->GetNextDataDir(opts, &dir));
- uint16_t uuid_idx;
+ int uuid_idx;
CHECK(dd_manager_->FindUuidIndexByDataDir(dir, &uuid_idx));
string path;
@@ -819,9 +822,9 @@ Status FileBlockManager::DeleteBlock(const BlockId& block_id) {
CHECK(!read_only_);
// Return early if deleting a block in a failed directory.
- set<uint16_t> failed_dirs = dd_manager_->GetFailedDataDirs();
+ set<int> failed_dirs = dd_manager_->GetFailedDataDirs();
if (PREDICT_FALSE(!failed_dirs.empty())) {
- uint16_t uuid_idx = internal::FileBlockLocation::GetDataDirIdx(block_id);
+ int uuid_idx = internal::FileBlockLocation::GetDataDirIdx(block_id);
if (ContainsKey(failed_dirs, uuid_idx)) {
LOG_EVERY_N(INFO, 10) << Substitute("Block $0 is in a failed directory; not deleting",
block_id.ToString());
http://git-wip-us.apache.org/repos/asf/kudu/blob/f96ab36a/src/kudu/fs/fs_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/fs_manager.cc b/src/kudu/fs/fs_manager.cc
index 4457ab0..a68769c 100644
--- a/src/kudu/fs/fs_manager.cc
+++ b/src/kudu/fs/fs_manager.cc
@@ -345,7 +345,7 @@ Status FsManager::Open(FsReport* report) {
// Before returning, ensure the metadata directory has not failed.
// TODO(awong): avoid this single point of failure by spreading metadata
// across directories.
- uint16_t metadata_idx;
+ int metadata_idx;
CHECK(dd_manager_->FindUuidIndexByRoot(canonicalized_metadata_fs_root_.path, &metadata_idx));
if (ContainsKey(dd_manager_->GetFailedDataDirs(), metadata_idx)) {
return Status::IOError("Cannot open the FS layout; metadata directory failed");
http://git-wip-us.apache.org/repos/asf/kudu/blob/f96ab36a/src/kudu/fs/log_block_manager-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/log_block_manager-test.cc b/src/kudu/fs/log_block_manager-test.cc
index 5dc5099..9a23c0c 100644
--- a/src/kudu/fs/log_block_manager-test.cc
+++ b/src/kudu/fs/log_block_manager-test.cc
@@ -1417,10 +1417,10 @@ TEST_F(LogBlockManagerTest, TestOpenWithFailedDirectories) {
for (const string& data_dir : report.data_dirs) {
ASSERT_NE(data_dir, test_dirs[failed_idx]);
}
- const set<uint16_t>& failed_dirs = dd_manager_->GetFailedDataDirs();
+ const set<int>& failed_dirs = dd_manager_->GetFailedDataDirs();
ASSERT_EQ(1, failed_dirs.size());
- uint16_t uuid_idx;
+ int uuid_idx;
dd_manager_->FindUuidIndexByRoot(test_dirs[failed_idx], &uuid_idx);
ASSERT_TRUE(ContainsKey(failed_dirs, uuid_idx));
FLAGS_env_inject_eio = 0;
http://git-wip-us.apache.org/repos/asf/kudu/blob/f96ab36a/src/kudu/fs/log_block_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/log_block_manager.cc b/src/kudu/fs/log_block_manager.cc
index 5f3d365..b9a9322 100644
--- a/src/kudu/fs/log_block_manager.cc
+++ b/src/kudu/fs/log_block_manager.cc
@@ -1724,7 +1724,7 @@ Status LogBlockManager::Open(FsReport* report) {
int i = -1;
for (const auto& dd : dd_manager_->data_dirs()) {
i++;
- uint16_t uuid_idx;
+ int uuid_idx;
CHECK(dd_manager_->FindUuidIndexByDataDir(dd.get(), &uuid_idx));
// TODO(awong): store Statuses for each directory in the directory manager
// so we can avoid these artificial Statuses.
@@ -1841,9 +1841,9 @@ Status LogBlockManager::DeleteBlock(const BlockId& block_id) {
}
// Return early if deleting a block in a failed directory.
- set<uint16_t> failed_dirs = dd_manager_->GetFailedDataDirs();
+ set<int> failed_dirs = dd_manager_->GetFailedDataDirs();
if (PREDICT_FALSE(!failed_dirs.empty())) {
- uint16_t uuid_idx;
+ int uuid_idx;
CHECK(dd_manager_->FindUuidIndexByDataDir(lb->container()->data_dir(), &uuid_idx));
if (ContainsKey(failed_dirs, uuid_idx)) {
LOG_EVERY_N(INFO, 10) << Substitute("Block $0 is in a failed directory; not deleting",
http://git-wip-us.apache.org/repos/asf/kudu/blob/f96ab36a/src/kudu/tserver/ts_tablet_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/ts_tablet_manager.cc b/src/kudu/tserver/ts_tablet_manager.cc
index 2967fe7..ceb3641 100644
--- a/src/kudu/tserver/ts_tablet_manager.cc
+++ b/src/kudu/tserver/ts_tablet_manager.cc
@@ -1276,7 +1276,7 @@ Status TSTabletManager::DeleteTabletData(
void TSTabletManager::FailTabletsInDataDir(const string& uuid) {
DataDirManager* dd_manager = fs_manager_->dd_manager();
- uint16_t uuid_idx;
+ int uuid_idx;
CHECK(dd_manager->FindUuidIndexByUuid(uuid, &uuid_idx))
<< Substitute("No data directory found with UUID $0", uuid);
LOG(FATAL) << Substitute("Data directory $0 failed. Disk failure handling not implemented",