You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by to...@apache.org on 2017/09/15 06:03:04 UTC
kudu git commit: log_block_manager: use move semantics to fill in the
block map
Repository: kudu
Updated Branches:
refs/heads/master 3c26cc3c2 -> 0f0f54eef
log_block_manager: use move semantics to fill in the block map
This avoids an extra ref-count increment/decrement while loading
the block map during startup. This substantially improves startup
time.
This shaved off another few percent from LBM startup time.
On a real host with 11M blocks:
Before:
I0907 17:20:42.277474 10021 fs_manager.cc:335] Time spent opening block manager: real 14.348s user 0.000s sys 0.001s
After:
I0907 17:27:13.600186 14550 fs_manager.cc:335] Time spent opening block manager: real 13.401s user 0.000s sys 0.002s
According to the benchmark:
Before:
I0907 17:15:50.645310 20302 log_block_manager-test.cc:799] Time spent reopening block manager: real 1.570s user 0.034s sys 0.001s
I0907 17:15:52.195543 20302 log_block_manager-test.cc:799] Time spent reopening block manager: real 1.550s user 0.037s sys 0.001s
I0907 17:15:53.755209 20302 log_block_manager-test.cc:799] Time spent reopening block manager: real 1.560s user 0.037s sys 0.001s
I0907 17:15:55.263762 20302 log_block_manager-test.cc:799] Time spent reopening block manager: real 1.509s user 0.038s sys 0.001s
I0907 17:15:56.818748 20302 log_block_manager-test.cc:799] Time spent reopening block manager: real 1.555s user 0.037s sys 0.001s
I0907 17:15:58.379680 20302 log_block_manager-test.cc:799] Time spent reopening block manager: real 1.561s user 0.036s sys 0.001s
I0907 17:15:59.913751 20302 log_block_manager-test.cc:799] Time spent reopening block manager: real 1.534s user 0.038s sys 0.000s
I0907 17:16:01.461668 20302 log_block_manager-test.cc:799] Time spent reopening block manager: real 1.548s user 0.037s sys 0.001s
I0907 17:16:03.020823 20302 log_block_manager-test.cc:799] Time spent reopening block manager: real 1.559s user 0.037s sys 0.001s
I0907 17:16:04.549747 20302 log_block_manager-test.cc:799] Time spent reopening block manager: real 1.529s user 0.035s sys 0.001s
After:
I0907 17:07:40.230087 9782 log_block_manager-test.cc:799] Time spent reopening block manager: real 1.495s user 0.032s sys 0.001s
I0907 17:07:41.642105 9782 log_block_manager-test.cc:799] Time spent reopening block manager: real 1.412s user 0.044s sys 0.001s
I0907 17:07:43.082852 9782 log_block_manager-test.cc:799] Time spent reopening block manager: real 1.441s user 0.040s sys 0.001s
I0907 17:07:44.512603 9782 log_block_manager-test.cc:799] Time spent reopening block manager: real 1.430s user 0.043s sys 0.000s
I0907 17:07:45.956848 9782 log_block_manager-test.cc:799] Time spent reopening block manager: real 1.444s user 0.041s sys 0.000s
I0907 17:07:47.386735 9782 log_block_manager-test.cc:799] Time spent reopening block manager: real 1.430s user 0.037s sys 0.002s
I0907 17:07:48.787317 9782 log_block_manager-test.cc:799] Time spent reopening block manager: real 1.401s user 0.041s sys 0.002s
I0907 17:07:50.244407 9782 log_block_manager-test.cc:799] Time spent reopening block manager: real 1.457s user 0.038s sys 0.001s
I0907 17:07:51.682209 9782 log_block_manager-test.cc:799] Time spent reopening block manager: real 1.438s user 0.037s sys 0.001s
I0907 17:07:53.116209 9782 log_block_manager-test.cc:799] Time spent reopening block manager: real 1.434s user 0.037s sys 0.000s
Change-Id: I8fe65087585e118f91320ec39f537b5f3ad6552f
Reviewed-on: http://gerrit.cloudera.org:8080/8008
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/0f0f54ee
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/0f0f54ee
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/0f0f54ee
Branch: refs/heads/master
Commit: 0f0f54eef51ad822bb937175ea14bf2a5f5491e3
Parents: 3c26cc3
Author: Todd Lipcon <to...@apache.org>
Authored: Thu Sep 7 12:52:52 2017 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Fri Sep 15 06:02:16 2017 +0000
----------------------------------------------------------------------
src/kudu/fs/log_block_manager.cc | 18 +++++++++++++-----
src/kudu/fs/log_block_manager.h | 2 +-
2 files changed, 14 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/0f0f54ee/src/kudu/fs/log_block_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/log_block_manager.cc b/src/kudu/fs/log_block_manager.cc
index 38e853a..4db8f15 100644
--- a/src/kudu/fs/log_block_manager.cc
+++ b/src/kudu/fs/log_block_manager.cc
@@ -1934,10 +1934,15 @@ scoped_refptr<LogBlock> LogBlockManager::AddLogBlock(
return nullptr;
}
-bool LogBlockManager::AddLogBlockUnlocked(const scoped_refptr<LogBlock>& lb) {
+bool LogBlockManager::AddLogBlockUnlocked(scoped_refptr<LogBlock> lb) {
DCHECK(lock_.is_locked());
- if (!InsertIfNotPresent(&blocks_by_block_id_, lb->block_id(), lb)) {
+ // InsertIfNotPresent doesn't use move semantics, so instead we just
+ // insert an empty scoped_refptr and assign into it down below rather
+ // than using the utility function.
+ scoped_refptr<LogBlock>* entry_ptr = &blocks_by_block_id_[lb->block_id()];
+ if (*entry_ptr) {
+ // Already have an entry for this block ID.
return false;
}
@@ -1951,6 +1956,8 @@ bool LogBlockManager::AddLogBlockUnlocked(const scoped_refptr<LogBlock>& lb) {
metrics()->blocks_under_management->Increment();
metrics()->bytes_under_management->IncrementBy(lb->length());
}
+
+ *entry_ptr = std::move(lb);
return true;
}
@@ -2200,14 +2207,15 @@ void LogBlockManager::OpenDataDir(DataDir* dir,
// memory in a local and add it to the mem-tracker in a single increment
// at the end of this loop.
int64_t mem_usage = 0;
- for (const UntrackedBlockMap::value_type& e : live_blocks) {
- if (!AddLogBlockUnlocked(e.second)) {
+ for (UntrackedBlockMap::value_type& e : live_blocks) {
+ int block_mem = kudu_malloc_usable_size(e.second.get());
+ if (!AddLogBlockUnlocked(std::move(e.second))) {
// TODO(adar): track as an inconsistency?
LOG(FATAL) << "Found duplicate CREATE record for block " << e.first
<< " which already is alive from another container when "
<< " processing container " << container->ToString();
}
- mem_usage += kudu_malloc_usable_size(e.second.get());
+ mem_usage += block_mem;
}
mem_tracker_->Consume(mem_usage);
http://git-wip-us.apache.org/repos/asf/kudu/blob/0f0f54ee/src/kudu/fs/log_block_manager.h
----------------------------------------------------------------------
diff --git a/src/kudu/fs/log_block_manager.h b/src/kudu/fs/log_block_manager.h
index bc38e5a..7b9b26a 100644
--- a/src/kudu/fs/log_block_manager.h
+++ b/src/kudu/fs/log_block_manager.h
@@ -284,7 +284,7 @@ class LogBlockManager : public BlockManager {
// Must hold 'lock_'.
//
// Returns true if the LogBlock was successfully added, false if it was already present.
- bool AddLogBlockUnlocked(const scoped_refptr<internal::LogBlock>& lb);
+ bool AddLogBlockUnlocked(scoped_refptr<internal::LogBlock> lb);
// Removes the given LogBlock from in-memory data structures. Must hold 'lock_'.
void RemoveLogBlockUnlocked(const BlockMap::iterator& it);