You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/05/03 15:22:47 UTC
[incubator-doris] branch dev-1.0.1 updated: [hotfix] track all memory in VOlapScanNode and fix track leaf from bitmap and hll (#9348)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/dev-1.0.1 by this push:
new 19924dfc80 [hotfix] track all memory in VOlapScanNode and fix track leaf from bitmap and hll (#9348)
19924dfc80 is described below
commit 19924dfc801b1c590018e9879c6613988efcd01e
Author: dataroaring <98...@users.noreply.github.com>
AuthorDate: Tue May 3 23:22:39 2022 +0800
[hotfix] track all memory in VOlapScanNode and fix track leaf from bitmap and hll (#9348)
This is a hotfix for dev-1.0.1, because the master branch has fixed this bug when refactoring memtracker
---
be/src/exec/olap_scan_node.cpp | 11 ++++-------
be/src/exec/olap_scan_node.h | 2 +-
be/src/olap/aggregate_func.h | 3 ---
be/src/vec/exec/volap_scan_node.cpp | 1 +
4 files changed, 6 insertions(+), 11 deletions(-)
diff --git a/be/src/exec/olap_scan_node.cpp b/be/src/exec/olap_scan_node.cpp
index 77991e5f39..d021cc3930 100644
--- a/be/src/exec/olap_scan_node.cpp
+++ b/be/src/exec/olap_scan_node.cpp
@@ -348,8 +348,7 @@ Status OlapScanNode::get_next(RuntimeState* state, RowBatch* row_batch, bool* eo
<< Tuple::to_string(row->get_tuple(0), *_tuple_desc);
}
}
- __sync_fetch_and_sub(&_buffered_bytes,
- row_batch->tuple_data_pool()->total_reserved_bytes());
+ _buffered_bytes -= row_batch->tuple_data_pool()->total_reserved_bytes();
delete materialized_batch;
return Status::OK();
@@ -1376,7 +1375,7 @@ void OlapScanNode::transfer_thread(RuntimeState* state) {
int64_t mem_limit = 512 * 1024 * 1024;
// TODO(zc): use memory limit
- int64_t mem_consume = __sync_fetch_and_add(&_buffered_bytes, 0);
+ int64_t mem_consume = _buffered_bytes;
if (state->fragment_mem_tracker() != nullptr) {
mem_limit = state->fragment_mem_tracker()->limit();
mem_consume = state->fragment_mem_tracker()->consumption();
@@ -1400,10 +1399,9 @@ void OlapScanNode::transfer_thread(RuntimeState* state) {
{
std::unique_lock<std::mutex> l(_scan_batches_lock);
assigned_thread_num = _running_thread;
- // int64_t buf_bytes = __sync_fetch_and_add(&_buffered_bytes, 0);
// How many thread can apply to this query
size_t thread_slot_num = 0;
- mem_consume = __sync_fetch_and_add(&_buffered_bytes, 0);
+ mem_consume = _buffered_bytes;
if (state->fragment_mem_tracker() != nullptr) {
mem_consume = state->fragment_mem_tracker()->consumption();
}
@@ -1617,8 +1615,7 @@ void OlapScanNode::scanner_thread(OlapScanner* scanner) {
row_batch = nullptr;
} else {
row_batchs.push_back(row_batch);
- __sync_fetch_and_add(&_buffered_bytes,
- row_batch->tuple_data_pool()->total_reserved_bytes());
+ _buffered_bytes += row_batch->tuple_data_pool()->total_reserved_bytes();
raw_bytes_read += row_batch->tuple_data_pool()->total_reserved_bytes();
}
raw_rows_read = scanner->raw_rows_read();
diff --git a/be/src/exec/olap_scan_node.h b/be/src/exec/olap_scan_node.h
index cb41822f30..1220a742e0 100644
--- a/be/src/exec/olap_scan_node.h
+++ b/be/src/exec/olap_scan_node.h
@@ -249,7 +249,7 @@ protected:
TResourceInfo* _resource_info;
- int64_t _buffered_bytes;
+ std::atomic<int64_t> _buffered_bytes;
EvalConjunctsFn _eval_conjuncts_fn;
bool _need_agg_finalize = true;
diff --git a/be/src/olap/aggregate_func.h b/be/src/olap/aggregate_func.h
index 02de11b866..abe3c41adc 100644
--- a/be/src/olap/aggregate_func.h
+++ b/be/src/olap/aggregate_func.h
@@ -505,8 +505,6 @@ struct AggregateFuncTraits<OLAP_FIELD_AGGREGATION_HLL_UNION, OLAP_FIELD_TYPE_HLL
dst_slice->data = reinterpret_cast<char*>(hll);
- mem_pool->mem_tracker()->Consume(hll->memory_consumed());
-
agg_pool->add(hll);
}
@@ -551,7 +549,6 @@ struct AggregateFuncTraits<OLAP_FIELD_AGGREGATION_BITMAP_UNION, OLAP_FIELD_TYPE_
// we use zero size represent this slice is a agg object
dst_slice->size = 0;
auto bitmap = new BitmapValue(src_slice->data);
- mem_pool->mem_tracker()->Consume(sizeof(BitmapValue));
dst_slice->data = (char*)bitmap;
agg_pool->add(bitmap);
diff --git a/be/src/vec/exec/volap_scan_node.cpp b/be/src/vec/exec/volap_scan_node.cpp
index f6101692ce..3156324d66 100644
--- a/be/src/vec/exec/volap_scan_node.cpp
+++ b/be/src/vec/exec/volap_scan_node.cpp
@@ -543,6 +543,7 @@ Block* VOlapScanNode::_alloc_block(bool& get_free_block) {
get_free_block = false;
auto block = new Block(_tuple_desc->slots(), _block_size);
+ _mem_tracker->Consume(block->allocated_bytes());
_buffered_bytes += block->allocated_bytes();
return block;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org