You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by da...@apache.org on 2022/07/30 02:53:46 UTC
[doris] branch master updated: [feature-wip](unique-key-merge-on-write) fix IndexedColumnReader and add primary key index memory tracker, DSIP-018 (#11303)
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1ada6d8641 [feature-wip](unique-key-merge-on-write) fix IndexedColumnReader and add primary key index memory tracker, DSIP-018 (#11303)
1ada6d8641 is described below
commit 1ada6d86414b17178c4a691003b4f6fba565340d
Author: Xin Liao <li...@126.com>
AuthorDate: Sat Jul 30 10:53:40 2022 +0800
[feature-wip](unique-key-merge-on-write) fix IndexedColumnReader and add primary key index memory tracker, DSIP-018 (#11303)
---
be/src/olap/primary_key_index.cpp | 5 +++--
be/src/olap/primary_key_index.h | 7 +++++--
be/src/olap/rowset/segment_v2/indexed_column_reader.cpp | 13 ++++++++++++-
be/src/olap/rowset/segment_v2/indexed_column_reader.h | 2 ++
be/src/olap/rowset/segment_v2/segment.cpp | 5 ++++-
5 files changed, 26 insertions(+), 6 deletions(-)
diff --git a/be/src/olap/primary_key_index.cpp b/be/src/olap/primary_key_index.cpp
index 92911b3ed1..52129675e7 100644
--- a/be/src/olap/primary_key_index.cpp
+++ b/be/src/olap/primary_key_index.cpp
@@ -17,6 +17,7 @@
#include "olap/primary_key_index.h"
+#include "common/config.h"
#include "io/fs/file_reader.h"
#include "olap/rowset/segment_v2/encoding_info.h"
@@ -67,13 +68,13 @@ Status PrimaryKeyIndexReader::parse(io::FileReaderSPtr file_reader,
const segment_v2::PrimaryKeyIndexMetaPB& meta) {
// parse primary key index
_index_reader.reset(new segment_v2::IndexedColumnReader(file_reader, meta.primary_key_index()));
- RETURN_IF_ERROR(_index_reader->load(_use_page_cache, _kept_in_memory));
+ RETURN_IF_ERROR(_index_reader->load(!config::disable_storage_page_cache, false));
// parse bloom filter
segment_v2::ColumnIndexMetaPB column_index_meta = meta.bloom_filter_index();
segment_v2::BloomFilterIndexReader bf_index_reader(std::move(file_reader),
&column_index_meta.bloom_filter_index());
- RETURN_IF_ERROR(bf_index_reader.load(_use_page_cache, _kept_in_memory));
+ RETURN_IF_ERROR(bf_index_reader.load(!config::disable_storage_page_cache, false));
std::unique_ptr<segment_v2::BloomFilterIndexIterator> bf_iter;
RETURN_IF_ERROR(bf_index_reader.new_iterator(&bf_iter));
RETURN_IF_ERROR(bf_iter->read_bloom_filter(0, &_bf));
diff --git a/be/src/olap/primary_key_index.h b/be/src/olap/primary_key_index.h
index 5645ba3c21..ac85ff64c8 100644
--- a/be/src/olap/primary_key_index.h
+++ b/be/src/olap/primary_key_index.h
@@ -93,10 +93,13 @@ public:
return _index_reader->num_values();
}
+ uint64_t get_memory_size() {
+ DCHECK(_parsed);
+ return _index_reader->get_memory_size() + _bf->size();
+ }
+
private:
bool _parsed;
- bool _use_page_cache = true;
- bool _kept_in_memory = true;
std::unique_ptr<segment_v2::IndexedColumnReader> _index_reader;
std::unique_ptr<segment_v2::BloomFilter> _bf;
};
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
index d1b39fb031..fb48833e3b 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
@@ -74,6 +74,7 @@ Status IndexedColumnReader::load_index_page(const PagePointerPB& pp, PageHandle*
RETURN_IF_ERROR(read_page(PagePointer(pp), handle, &body, &footer, INDEX_PAGE,
local_compress_codec.get()));
RETURN_IF_ERROR(reader->parse(body, footer.index_page_footer()));
+ _mem_size += body.get_size();
return Status::OK();
}
@@ -194,7 +195,17 @@ Status IndexedColumnIterator::seek_at_or_after(const void* key, bool* exact_matc
}
// seek inside data page
- RETURN_IF_ERROR(_data_page.data_decoder->seek_at_or_after_value(key, exact_match));
+ Status st = _data_page.data_decoder->seek_at_or_after_value(key, exact_match);
+ // return the first row of next page when not found
+ if (st.is_not_found() && _reader->_has_index_page) {
+ if (_value_iter.move_next()) {
+ _seeked = true;
+ *exact_match = false;
+ _current_ordinal = _data_page.first_ordinal + _data_page.num_rows;
+ return Status::OK();
+ }
+ }
+ RETURN_IF_ERROR(st);
_data_page.offset_in_page = _data_page.data_decoder->current_index();
_current_ordinal = _data_page.first_ordinal + _data_page.offset_in_page;
DCHECK(_data_page.contains(_current_ordinal));
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.h b/be/src/olap/rowset/segment_v2/indexed_column_reader.h
index 8ce9f63d3b..e70b2aae5c 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.h
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.h
@@ -63,6 +63,7 @@ public:
bool support_value_seek() const { return _meta.has_value_index_meta(); }
CompressionTypePB get_compression() const { return _meta.compression(); }
+ uint64_t get_memory_size() const { return _mem_size; }
private:
Status load_index_page(const PagePointerPB& pp, PageHandle* handle, IndexPageReader* reader);
@@ -88,6 +89,7 @@ private:
const TypeInfo* _type_info = nullptr;
const EncodingInfo* _encoding_info = nullptr;
const KeyCoder* _value_key_coder = nullptr;
+ uint64_t _mem_size = 0;
};
class IndexedColumnIterator {
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp
index 6b84cd9566..892743a63d 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -147,7 +147,10 @@ Status Segment::load_index() {
if (_tablet_schema.keys_type() == UNIQUE_KEYS && _footer.has_primary_key_index_meta()) {
_pk_index_reader.reset(new PrimaryKeyIndexReader());
- return _pk_index_reader->parse(_file_reader, _footer.primary_key_index_meta());
+ RETURN_IF_ERROR(
+ _pk_index_reader->parse(_file_reader, _footer.primary_key_index_meta()));
+ _meta_mem_usage += _pk_index_reader->get_memory_size();
+ return Status::OK();
} else {
Slice body;
PageFooterPB footer;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org