You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by we...@apache.org on 2022/06/21 12:55:38 UTC
[doris] 05/05: Revert "[wx-opt-mem-feature] add profile in memtable"
This is an automated email from the ASF dual-hosted git repository.
weixiang pushed a commit to branch memtable_opt_rebase_bak
in repository https://gitbox.apache.org/repos/asf/doris.git
commit c385442d79578808619aef5c7649bf22051be83e
Author: spaces-x <we...@gmail.com>
AuthorDate: Tue Jun 21 20:54:42 2022 +0800
Revert "[wx-opt-mem-feature] add profile in memtable"
This reverts commit bf98cd8c7efa74242970d421b1174b8eaec85059.
---
be/src/olap/memtable.cpp | 151 +++++++++++++++++++++--------------------------
be/src/olap/memtable.h | 24 +-------
2 files changed, 69 insertions(+), 106 deletions(-)
diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp
index eebaed0baf..b7d1514688 100644
--- a/be/src/olap/memtable.cpp
+++ b/be/src/olap/memtable.cpp
@@ -55,7 +55,6 @@ MemTable::MemTable(int64_t tablet_id, Schema* schema, const TabletSchema* tablet
_vec_row_comparator.get(), _table_mem_pool.get(), _keys_type == KeysType::DUP_KEYS);
_block_aggregator =
std::make_unique<vectorized::BlockAggregator>(_schema, _tablet_schema, true);
- _init_profile();
} else {
_vec_skip_list = nullptr;
if (_keys_type == KeysType::DUP_KEYS) {
@@ -101,7 +100,6 @@ void MemTable::_init_agg_functions(const vectorized::Block* block) {
MemTable::~MemTable() {
std::for_each(_row_in_blocks.begin(), _row_in_blocks.end(), std::default_delete<RowInBlock>());
_mem_tracker->release(_mem_usage);
- print_profile();
}
MemTable::RowCursorComparator::RowCursorComparator(const Schema* schema) : _schema(schema) {}
@@ -119,35 +117,31 @@ int MemTable::RowInBlockComparator::operator()(const RowInBlock* left,
}
bool MemTable::insert(const vectorized::Block* block, size_t row_pos, size_t num_rows) {
- {
- SCOPED_TIMER(_insert_time);
- if (_is_first_insertion) {
- _is_first_insertion = false;
- auto cloneBlock = block->clone_without_columns();
- _block = std::make_shared<vectorized::MutableBlock>(&cloneBlock);
- if (_keys_type != KeysType::DUP_KEYS) {
- _init_agg_functions(block);
- }
+ if (_is_first_insertion) {
+ _is_first_insertion = false;
+ auto cloneBlock = block->clone_without_columns();
+ _block = std::make_shared<vectorized::MutableBlock>(&cloneBlock);
+ if (_keys_type != KeysType::DUP_KEYS) {
+ _init_agg_functions(block);
}
- _block->add_rows(block, row_pos, num_rows);
- _block_bytes_usage += block->allocated_bytes() * num_rows / block->rows();
- // Memtalbe is full, do not flush immediately
- // First try to merge these blocks
- // If the merged memtable is still full or we can not benefit a lot from merge at first
- // Then flush the memtable into disk.
- bool is_flush = false;
- if (is_full()) {
- size_t before_merge_bytes = bytes_allocated();
- _merge();
- size_t after_merged_bytes = bytes_allocated();
- // TODO(weixiang): magic number here, make it configurable later.
- if (is_full() ||
- (after_merged_bytes >= before_merge_bytes * 2 / 3 && _merge_count == 1)) {
- is_flush = true;
- }
+ }
+ _block->add_rows(block, row_pos, num_rows);
+ _block_bytes_usage += block->allocated_bytes() * num_rows / block->rows();
+ // Memtalbe is full, do not flush immediately
+ // First try to merge these blocks
+ // If the merged memtable is still full or we can not benefit a lot from merge at first
+ // Then flush the memtable into disk.
+ bool is_flush = false;
+ if (is_full()) {
+ size_t before_merge_bytes = bytes_allocated();
+ _merge();
+ size_t after_merged_bytes = bytes_allocated();
+ // TODO(weixiang): magic number here, make it configurable later.
+ if (is_full() || (after_merged_bytes >= before_merge_bytes * 2 / 3 && _merge_count == 1)) {
+ is_flush = true;
}
- return is_flush;
}
+ return is_flush;
}
size_t MemTable::bytes_allocated() const {
@@ -168,41 +162,35 @@ void MemTable::_merge() {
}
void MemTable::_agg(const bool finalize) {
- {
- SCOPED_TIMER(_agg_time);
- // note that the _block had been sorted before.
- if (_sorted_block == nullptr || _sorted_block->rows() <= 0) {
- return;
- }
- vectorized::Block sorted_block = _sorted_block->to_block();
- _block_aggregator->append_block(&sorted_block);
- _block_aggregator->partial_sort_merged_aggregate();
- if (finalize) {
- _sorted_block.reset();
- } else {
- _sorted_block->clear_column_data();
- }
+ // note that the _block had been sorted before.
+ if (_sorted_block == nullptr || _sorted_block->rows() <= 0) {
+ return;
+ }
+ vectorized::Block sorted_block = _sorted_block->to_block();
+ _block_aggregator->append_block(&sorted_block);
+ _block_aggregator->partial_sort_merged_aggregate();
+ if (finalize) {
+ _sorted_block.reset();
+ } else {
+ _sorted_block->clear_column_data();
}
}
void MemTable::_sort(const bool finalize) {
- {
- SCOPED_TIMER(_sort_time);
- _index_for_sort.resize(_block->rows());
- for (uint32_t i = 0; i < _block->rows(); i++) {
- _index_for_sort[i] = {i, i};
- }
+ _index_for_sort.resize(_block->rows());
+ for (uint32_t i = 0; i < _block->rows(); i++) {
+ _index_for_sort[i] = {i, i};
+ }
- _sort_block_by_rows();
- _sorted_block = _block->create_same_struct_block(0);
- _append_sorted_block(_block.get(), _sorted_block.get());
- if (finalize) {
- _block.reset();
- } else {
- _block->clear_column_data();
- }
- _block_bytes_usage = 0;
+ _sort_block_by_rows();
+ _sorted_block = _block->create_same_struct_block(0);
+ _append_sorted_block(_block.get(), _sorted_block.get());
+ if (finalize) {
+ _block.reset();
+ } else {
+ _block->clear_column_data();
}
+ _block_bytes_usage = 0;
}
void MemTable::_sort_block_by_rows() {
@@ -229,35 +217,32 @@ void MemTable::_append_sorted_block(vectorized::MutableBlock* src, vectorized::M
}
void MemTable::finalize() {
- {
- SCOPED_TIMER(_finalize_time);
- //TODO(weixiang): check here
- if (_block == nullptr) {
- return;
- }
-
- if (_keys_type != KeysType::DUP_KEYS) {
- // agg mode
- if (_block->rows() > 0) {
- _merge();
- }
- if (_merge_count > 1) {
- _block = _block_aggregator->get_partial_agged_block();
- _block_aggregator->reset_aggregator();
- _sort(true);
- _agg(true);
- } else {
- _block.reset();
- _sorted_block.reset();
- }
-
- _block_bytes_usage = 0;
- _sorted_block = _block_aggregator->get_partial_agged_block();
+ //TODO(weixiang): check here
+ if (_block == nullptr) {
+ return;
+ }
- } else {
- // dup mode
+ if (_keys_type != KeysType::DUP_KEYS) {
+ // agg mode
+ if (_block->rows() > 0) {
+ _merge();
+ }
+ if (_merge_count > 1) {
+ _block = _block_aggregator->get_partial_agged_block();
+ _block_aggregator->reset_aggregator();
_sort(true);
+ _agg(true);
+ } else {
+ _block.reset();
+ _sorted_block.reset();
}
+
+ _block_bytes_usage = 0;
+ _sorted_block = _block_aggregator->get_partial_agged_block();
+
+ } else {
+ // dup mode
+ _sort(true);
}
}
diff --git a/be/src/olap/memtable.h b/be/src/olap/memtable.h
index 37e38d7b56..d7bdf3b260 100644
--- a/be/src/olap/memtable.h
+++ b/be/src/olap/memtable.h
@@ -24,12 +24,11 @@
#include "olap/skiplist.h"
#include "runtime/mem_tracker.h"
#include "util/pdqsort.h"
-#include "util/runtime_profile.h"
#include "util/tuple_row_zorder_compare.h"
#include "vec/aggregate_functions/aggregate_function.h"
+#include "vec/aggregate_functions/block_aggregator.h"
#include "vec/common/string_ref.h"
#include "vec/core/block.h"
-#include "vec/aggregate_functions/block_aggregator.h"
namespace doris {
@@ -164,20 +163,6 @@ private:
void _append_sorted_block(vectorized::MutableBlock* src, vectorized::MutableBlock* dst);
- void _init_profile() {
- _profile.reset(new RuntimeProfile("Memtable"));
- _insert_time = ADD_TIMER(_profile, "insert time");
- _sort_time = ADD_TIMER(_profile, "sort time");
- _agg_time = ADD_TIMER(_profile, "agg time");
- _finalize_time = ADD_TIMER(_profile, "finalize time");
- }
-
- void print_profile() {
- std::stringstream ss;
- _profile->pretty_print(&ss);
- LOG(INFO) << ss.str();
- }
-
int64_t _tablet_id;
Schema* _schema;
const TabletSchema* _tablet_schema;
@@ -255,13 +240,6 @@ private:
size_t _block_bytes_usage = 0;
size_t _agg_bytes_usage = 0;
int _merge_count = 0;
-
- std::unique_ptr<RuntimeProfile> _profile;
- RuntimeProfile::Counter* _insert_time;
- RuntimeProfile::Counter* _sort_time;
- RuntimeProfile::Counter* _agg_time;
- RuntimeProfile::Counter* _finalize_time;
-
}; // class MemTable
inline std::ostream& operator<<(std::ostream& os, const MemTable& table) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org