You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by ad...@apache.org on 2019/10/02 04:11:19 UTC
[kudu] 02/05: cfile: track memory for CFileReaders and
BloomFileReaders
This is an automated email from the ASF dual-hosted git repository.
adar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 62c2fddcdc65f2092ee335ac1a35288dd3f7e4a5
Author: Andrew Wong <aw...@apache.org>
AuthorDate: Tue Oct 1 13:30:05 2019 -0700
cfile: track memory for CFileReaders and BloomFileReaders
This lumps a tablet's CFileReader tracked memory and BloomFileReader
tracked memory into the ids "CFileReaders" and "BloomFileReaders"
respectively.
I test this locally and verified via the web UI that I could see
CFileReader and BloomFileReader entries once they had been flushed.
Change-Id: Ic62c89079edffb41d1f5b40d7b699c525fa447a7
Reviewed-on: http://gerrit.cloudera.org:8080/14322
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Kudu Jenkins
---
src/kudu/tablet/cfile_set-test.cc | 15 ++++++++++-----
src/kudu/tablet/cfile_set.cc | 22 +++++++++++++---------
src/kudu/tablet/cfile_set.h | 9 ++++++---
src/kudu/tablet/diskrowset.cc | 6 ++++--
src/kudu/tablet/tablet_mem_trackers.h | 8 +++++++-
5 files changed, 40 insertions(+), 20 deletions(-)
diff --git a/src/kudu/tablet/cfile_set-test.cc b/src/kudu/tablet/cfile_set-test.cc
index e3fd517..c074e9d 100644
--- a/src/kudu/tablet/cfile_set-test.cc
+++ b/src/kudu/tablet/cfile_set-test.cc
@@ -278,7 +278,8 @@ TEST_F(TestCFileSet, TestPartiallyMaterialize) {
WriteTestRowSet(kNumRows);
shared_ptr<CFileSet> fileset;
- ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), nullptr, &fileset));
+ ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), MemTracker::GetRootTracker(),
+ nullptr, &fileset));
unique_ptr<CFileSet::Iterator> iter(fileset->NewIterator(&schema_, nullptr));
ASSERT_OK(iter->Init(nullptr));
@@ -358,7 +359,8 @@ TEST_F(TestCFileSet, TestIteratePartialSchema) {
WriteTestRowSet(kNumRows);
shared_ptr<CFileSet> fileset;
- ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), nullptr, &fileset));
+ ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), MemTracker::GetRootTracker(),
+ nullptr, &fileset));
Schema new_schema;
ASSERT_OK(schema_.CreateProjectionByNames({ "c0", "c2" }, &new_schema));
@@ -391,7 +393,8 @@ TEST_F(TestCFileSet, TestRangeScan) {
WriteTestRowSet(kNumRows);
shared_ptr<CFileSet> fileset;
- ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), nullptr, &fileset));
+ ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), MemTracker::GetRootTracker(),
+ nullptr, &fileset));
// Create iterator.
unique_ptr<CFileSet::Iterator> cfile_iter(fileset->NewIterator(&schema_, nullptr));
@@ -445,7 +448,8 @@ TEST_F(TestCFileSet, TestRangePredicates2) {
WriteTestRowSet(kNumRows);
shared_ptr<CFileSet> fileset;
- ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), nullptr, &fileset));
+ ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), MemTracker::GetRootTracker(),
+ nullptr, &fileset));
// Range scan where rows match on both ends
DoTestRangeScan(fileset, 2000, 2010);
@@ -501,7 +505,8 @@ TEST_F(TestCFileSet, TestBloomFilterPredicates) {
&ret1_contain, &ret1_exclude, &ret2_contain, &ret2_exclude);
shared_ptr<CFileSet> fileset;
- ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), nullptr, &fileset));
+ ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), MemTracker::GetRootTracker(),
+ nullptr, &fileset));
vector<ColumnPredicate::BloomFilterInner> bfs;
// BloomFilter of column 0 contain.
diff --git a/src/kudu/tablet/cfile_set.cc b/src/kudu/tablet/cfile_set.cc
index 52a892b..e1b7881 100644
--- a/src/kudu/tablet/cfile_set.cc
+++ b/src/kudu/tablet/cfile_set.cc
@@ -88,7 +88,7 @@ using strings::Substitute;
////////////////////////////////////////////////////////////
static Status OpenReader(FsManager* fs,
- shared_ptr<MemTracker> parent_mem_tracker,
+ shared_ptr<MemTracker> cfile_reader_tracker,
const BlockId& block_id,
const IOContext* io_context,
unique_ptr<CFileReader>* new_reader) {
@@ -96,7 +96,7 @@ static Status OpenReader(FsManager* fs,
RETURN_NOT_OK(fs->OpenBlock(block_id, &block));
ReaderOptions opts;
- opts.parent_mem_tracker = std::move(parent_mem_tracker);
+ opts.parent_mem_tracker = std::move(cfile_reader_tracker);
opts.io_context = io_context;
return CFileReader::OpenNoInit(std::move(block),
std::move(opts),
@@ -108,20 +108,24 @@ static Status OpenReader(FsManager* fs,
////////////////////////////////////////////////////////////
CFileSet::CFileSet(shared_ptr<RowSetMetadata> rowset_metadata,
- shared_ptr<MemTracker> parent_mem_tracker)
+ shared_ptr<MemTracker> bloomfile_tracker,
+ shared_ptr<MemTracker> cfile_reader_tracker)
: rowset_metadata_(std::move(rowset_metadata)),
- parent_mem_tracker_(std::move(parent_mem_tracker)) {
+ bloomfile_tracker_(std::move(bloomfile_tracker)),
+ cfile_reader_tracker_(std::move(cfile_reader_tracker)) {
}
CFileSet::~CFileSet() {
}
Status CFileSet::Open(shared_ptr<RowSetMetadata> rowset_metadata,
- shared_ptr<MemTracker> parent_mem_tracker,
+ shared_ptr<MemTracker> bloomfile_tracker,
+ shared_ptr<MemTracker> cfile_reader_tracker,
const IOContext* io_context,
shared_ptr<CFileSet>* cfile_set) {
shared_ptr<CFileSet> cfs(new CFileSet(std::move(rowset_metadata),
- std::move(parent_mem_tracker)));
+ std::move(bloomfile_tracker),
+ std::move(cfile_reader_tracker)));
RETURN_NOT_OK(cfs->DoOpen(io_context));
cfile_set->swap(cfs);
@@ -140,7 +144,7 @@ Status CFileSet::DoOpen(const IOContext* io_context) {
unique_ptr<CFileReader> reader;
RETURN_NOT_OK(OpenReader(rowset_metadata_->fs_manager(),
- parent_mem_tracker_,
+ cfile_reader_tracker_,
rowset_metadata_->column_data_block_for_col_id(col_id),
io_context,
&reader));
@@ -152,7 +156,7 @@ Status CFileSet::DoOpen(const IOContext* io_context) {
if (rowset_metadata_->has_adhoc_index_block()) {
RETURN_NOT_OK(OpenReader(rowset_metadata_->fs_manager(),
- parent_mem_tracker_,
+ cfile_reader_tracker_,
rowset_metadata_->adhoc_index_block(),
io_context,
&ad_hoc_idx_reader_));
@@ -183,8 +187,8 @@ Status CFileSet::OpenBloomReader(const IOContext* io_context) {
RETURN_NOT_OK(fs->OpenBlock(rowset_metadata_->bloom_block(), &block));
ReaderOptions opts;
- opts.parent_mem_tracker = parent_mem_tracker_;
opts.io_context = io_context;
+ opts.parent_mem_tracker = bloomfile_tracker_;
Status s = BloomFileReader::OpenNoInit(std::move(block),
std::move(opts),
&bloom_reader_);
diff --git a/src/kudu/tablet/cfile_set.h b/src/kudu/tablet/cfile_set.h
index 4721e95..a34dd21 100644
--- a/src/kudu/tablet/cfile_set.h
+++ b/src/kudu/tablet/cfile_set.h
@@ -74,7 +74,8 @@ class CFileSet : public std::enable_shared_from_this<CFileSet> {
class Iterator;
static Status Open(std::shared_ptr<RowSetMetadata> rowset_metadata,
- std::shared_ptr<MemTracker> parent_mem_tracker,
+ std::shared_ptr<MemTracker> bloomfile_tracker,
+ std::shared_ptr<MemTracker> cfile_reader_tracker,
const fs::IOContext* io_context,
std::shared_ptr<CFileSet>* cfile_set);
@@ -133,7 +134,8 @@ class CFileSet : public std::enable_shared_from_this<CFileSet> {
DISALLOW_COPY_AND_ASSIGN(CFileSet);
CFileSet(std::shared_ptr<RowSetMetadata> rowset_metadata,
- std::shared_ptr<MemTracker> parent_mem_tracker);
+ std::shared_ptr<MemTracker> bloomfile_tracker,
+ std::shared_ptr<MemTracker> cfile_reader_tracker);
Status DoOpen(const fs::IOContext* io_context);
Status OpenBloomReader(const fs::IOContext* io_context);
@@ -153,7 +155,8 @@ class CFileSet : public std::enable_shared_from_this<CFileSet> {
const Schema &tablet_schema() const { return rowset_metadata_->tablet_schema(); }
std::shared_ptr<RowSetMetadata> rowset_metadata_;
- std::shared_ptr<MemTracker> parent_mem_tracker_;
+ std::shared_ptr<MemTracker> bloomfile_tracker_;
+ std::shared_ptr<MemTracker> cfile_reader_tracker_;
std::string min_encoded_key_;
std::string max_encoded_key_;
diff --git a/src/kudu/tablet/diskrowset.cc b/src/kudu/tablet/diskrowset.cc
index b2395a9..f4dd37d 100644
--- a/src/kudu/tablet/diskrowset.cc
+++ b/src/kudu/tablet/diskrowset.cc
@@ -535,7 +535,8 @@ DiskRowSet::DiskRowSet(shared_ptr<RowSetMetadata> rowset_metadata,
Status DiskRowSet::Open(const IOContext* io_context) {
TRACE_EVENT0("tablet", "DiskRowSet::Open");
RETURN_NOT_OK(CFileSet::Open(rowset_metadata_,
- mem_trackers_.tablet_tracker,
+ mem_trackers_.bloomfile_tracker,
+ mem_trackers_.cfile_reader_tracker,
io_context,
&base_data_));
@@ -607,7 +608,8 @@ Status DiskRowSet::MajorCompactDeltaStoresWithColumnIds(const vector<ColumnId>&
// appropriate blocks to match the update.
shared_ptr<CFileSet> new_base;
RETURN_NOT_OK(CFileSet::Open(rowset_metadata_,
- mem_trackers_.tablet_tracker,
+ mem_trackers_.bloomfile_tracker,
+ mem_trackers_.cfile_reader_tracker,
io_context,
&new_base));
{
diff --git a/src/kudu/tablet/tablet_mem_trackers.h b/src/kudu/tablet/tablet_mem_trackers.h
index 4882318..405b4cb 100644
--- a/src/kudu/tablet/tablet_mem_trackers.h
+++ b/src/kudu/tablet/tablet_mem_trackers.h
@@ -30,6 +30,8 @@ struct TabletMemTrackers {
// Intended for unit tests where the tracker hierarchy doesn't matter.
TabletMemTrackers()
: tablet_tracker(MemTracker::GetRootTracker()),
+ bloomfile_tracker(MemTracker::GetRootTracker()),
+ cfile_reader_tracker(MemTracker::GetRootTracker()),
dms_tracker(MemTracker::GetRootTracker()) {
}
@@ -38,13 +40,17 @@ struct TabletMemTrackers {
: tablet_tracker(MemTracker::CreateTracker(
-1,
strings::Substitute("tablet-$0", tablet_id),
- parent_mem_tracker)),
+ std::move(parent_mem_tracker))),
+ bloomfile_tracker(MemTracker::CreateTracker(-1, "BloomFileReaders", tablet_tracker)),
+ cfile_reader_tracker(MemTracker::CreateTracker(-1, "CFileReaders", tablet_tracker)),
dms_tracker(MemTracker::CreateTracker(-1, "DeltaMemStores", tablet_tracker)) {
}
std::shared_ptr<MemTracker> tablet_tracker;
// All of the below are children of tablet_tracker;
+ std::shared_ptr<MemTracker> bloomfile_tracker;
+ std::shared_ptr<MemTracker> cfile_reader_tracker;
std::shared_ptr<MemTracker> dms_tracker;
};