[incubator-doris] branch master updated: Fix bug of memory limit when group by varchar columns. (#3242)

[zh...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

zhaoc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 6a9a629  Fix bug of memory limit when group by varchar columns. (#3242)
6a9a629 is described below

commit 6a9a62901faacb6a3738eca34c430a64c2bf387d
Author: lichaoyong <li...@baidu.com>
AuthorDate: Wed Apr 1 18:48:18 2020 +0800

    Fix bug of memory limit when group by varchar columns. (#3242)
    
    select date_format(k10, '%Y%m%d') as myk10 from baseall group by myk10;
    The date_format function in query above will be stored in MemPool during
    the query execution. If the query handles millions of rows, it will
    consume much memory. Should clear the MemPool at interval.
---
 be/src/exec/partitioned_aggregation_node_ir.cc | 2 ++
 be/src/exec/partitioned_hash_table.h           | 1 +
 2 files changed, 3 insertions(+)

diff --git a/be/src/exec/partitioned_aggregation_node_ir.cc b/be/src/exec/partitioned_aggregation_node_ir.cc
index b706bfe..7fa7bf8 100644
--- a/be/src/exec/partitioned_aggregation_node_ir.cc
+++ b/be/src/exec/partitioned_aggregation_node_ir.cc
@@ -56,6 +56,7 @@ Status PartitionedAggregationNode::ProcessBatch(RowBatch* batch,
       RETURN_IF_ERROR(ProcessRow<AGGREGATED_ROWS>(batch_iter.get(), ht_ctx));
       expr_vals_cache->NextRow();
     }
+    ht_ctx->expr_results_pool_->clear();
     DCHECK(expr_vals_cache->AtEnd());
   }
   return Status::OK();
@@ -199,6 +200,7 @@ Status PartitionedAggregationNode::ProcessBatchStreaming(bool needs_serialize,
       DCHECK(process_batch_status_.ok());
       expr_vals_cache->NextRow();
     }
+    ht_ctx->expr_results_pool_->clear();
     DCHECK(expr_vals_cache->AtEnd());
   }
   if (needs_serialize) {
diff --git a/be/src/exec/partitioned_hash_table.h b/be/src/exec/partitioned_hash_table.h
index 5b331cb..ef90ac6 100644
--- a/be/src/exec/partitioned_hash_table.h
+++ b/be/src/exec/partitioned_hash_table.h
@@ -358,6 +358,7 @@ class PartitionedHashTableCtx {
   ExprValuesCache* ALWAYS_INLINE expr_values_cache() { return &expr_values_cache_; }
 
  private:
+  friend class PartitionedAggregationNode;
   friend class PartitionedHashTable;
   friend class HashTableTest_HashEmpty_Test;
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org