You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/04/26 15:52:17 UTC
[incubator-doris] 14/15: [fix](OlapScanner)fix bitmap or hll's OOM when loading too many unqualified data (#9205)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 9cfed506d38c85867e8a9c5def4369235987309a
Author: SleepyBear <ka...@live.cn>
AuthorDate: Tue Apr 26 10:25:56 2022 +0800
[fix](OlapScanner)fix bitmap or hll's OOM when loading too many unqualified data (#9205)
---
be/src/common/config.h | 4 ++++
be/src/common/object_pool.h | 5 +++++
be/src/exec/olap_scanner.cpp | 17 ++++++++++++++++-
3 files changed, 25 insertions(+), 1 deletion(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 2a5f03de8e..b37fbc048a 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -711,6 +711,10 @@ CONF_mInt32(string_type_length_soft_limit_bytes, "1048576");
CONF_Validator(string_type_length_soft_limit_bytes,
[](const int config) -> bool { return config > 0 && config <= 2147483643; });
+// used for olap scanner to save memory, when the size of unused_object_pool
+// is greater than object_pool_buffer_size, release the object in the unused_object_pool.
+CONF_Int32(object_pool_buffer_size, "100");
+
} // namespace config
} // namespace doris
diff --git a/be/src/common/object_pool.h b/be/src/common/object_pool.h
index b2eb4e3024..50001c1737 100644
--- a/be/src/common/object_pool.h
+++ b/be/src/common/object_pool.h
@@ -60,6 +60,11 @@ public:
src->_objects.clear();
}
+ uint64_t size() {
+ std::lock_guard<SpinLock> l(_lock);
+ return _objects.size();
+ }
+
private:
DISALLOW_COPY_AND_ASSIGN(ObjectPool);
diff --git a/be/src/exec/olap_scanner.cpp b/be/src/exec/olap_scanner.cpp
index 04dfa81daf..9baef00001 100644
--- a/be/src/exec/olap_scanner.cpp
+++ b/be/src/exec/olap_scanner.cpp
@@ -272,6 +272,11 @@ Status OlapScanner::get_batch(RuntimeState* state, RowBatch* batch, bool* eof) {
int64_t raw_bytes_threshold = config::doris_scanner_row_bytes;
{
SCOPED_TIMER(_parent->_scan_timer);
+ // store the object which may can't pass the conjuncts temporarily.
+ // otherwise, pushed all objects into agg_object_pool directly may lead to OOM.
+ ObjectPool tmp_object_pool;
+ // release the memory of the object which can't pass the conjuncts.
+ ObjectPool unused_object_pool;
while (true) {
// Batch is full or reach raw_rows_threshold or raw_bytes_threshold, break
if (batch->is_full() ||
@@ -280,9 +285,18 @@ Status OlapScanner::get_batch(RuntimeState* state, RowBatch* batch, bool* eof) {
_update_realtime_counter();
break;
}
+
+ if (tmp_object_pool.size() > 0) {
+ unused_object_pool.acquire_data(&tmp_object_pool);
+ }
+
+ if (unused_object_pool.size() >= config::object_pool_buffer_size) {
+ unused_object_pool.clear();
+ }
+
// Read one row from reader
auto res = _tablet_reader->next_row_with_aggregation(&_read_row_cursor, mem_pool.get(),
- batch->agg_object_pool(), eof);
+ &tmp_object_pool, eof);
if (res != OLAP_SUCCESS) {
std::stringstream ss;
ss << "Internal Error: read storage fail. res=" << res
@@ -404,6 +418,7 @@ Status OlapScanner::get_batch(RuntimeState* state, RowBatch* batch, bool* eof) {
// check direct && pushdown conjuncts success then commit tuple
batch->commit_last_row();
+ batch->agg_object_pool()->acquire_data(&tmp_object_pool);
char* new_tuple = reinterpret_cast<char*>(tuple);
new_tuple += _tuple_desc->byte_size();
tuple = reinterpret_cast<Tuple*>(new_tuple);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org