[GitHub] [incubator-doris] morningman commented on a change in pull request #5811: [Enhancement] Reduce memory consumption by releasing readers earier

[GitBox <gi...@apache.org>]

morningman commented on a change in pull request #5811:
URL: https://github.com/apache/incubator-doris/pull/5811#discussion_r645927155



##########
File path: be/src/olap/rowset/alpha_rowset_reader.cpp
##########
@@ -120,20 +129,24 @@ int64_t AlphaRowsetReader::filtered_rows() {
 }
 
 OLAPStatus AlphaRowsetReader::_union_block(RowBlock** block) {
-    while (_ordinal < _merge_ctxs.size()) {
+    while (_cur_ctx != nullptr) {
         // union block only use one block to store
-        OLAPStatus status = _pull_next_block(&(_merge_ctxs[_ordinal]));
+        OLAPStatus status = _pull_next_block(_cur_ctx);
         if (status == OLAP_ERR_DATA_EOF) {
-            _ordinal++;
-            continue;
+            delete _cur_ctx;
+            _cur_ctx = nullptr;
+            _sequential_ctxs.pop_front();
+            if (!_sequential_ctxs.empty()) {
+                _cur_ctx = *(_sequential_ctxs.begin());
+            }
         } else if (status != OLAP_SUCCESS) {
             return status;
         } else {
-            (*block) = _merge_ctxs[_ordinal].row_block;
+            (*block) = _cur_ctx->row_block;
             return OLAP_SUCCESS;
         }
     }
-    if (_ordinal == _merge_ctxs.size()) {
+    if (_sequential_ctxs.empty()) {

Review comment:
       Seems that this can be a `CHECK(_init_merge_heap.empty())`

##########
File path: be/src/olap/rowset/alpha_rowset_reader.cpp
##########
@@ -37,6 +37,15 @@ AlphaRowsetReader::AlphaRowsetReader(int num_rows_per_row_block, AlphaRowsetShar
 AlphaRowsetReader::~AlphaRowsetReader() {
     delete _dst_cursor;
     _rowset->release();
+    while (!_merge_heap.empty()) {

Review comment:
       Why in previous implementation, we don't need to clear the `_merge_heap` here?

##########
File path: be/src/olap/rowset/unique_rowset_id_generator.h
##########
@@ -25,6 +25,7 @@
 
 namespace doris {
 
+// TODO(yingchun): why use two classes?

Review comment:
       rowset id maybe changed to another format with some other information, if necessary.




-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org