[GitHub] [incubator-doris] imay commented on a change in pull request #2359: Improve SkipList memory usage tracking

[GitBox <gi...@apache.org>]

imay commented on a change in pull request #2359: Improve SkipList memory usage tracking
URL: https://github.com/apache/incubator-doris/pull/2359#discussion_r353021046
 
 

 ##########
 File path: be/src/olap/memtable.cpp
 ##########
 @@ -61,26 +62,49 @@ int MemTable::RowCursorComparator::operator()(const char* left, const char* righ
     return compare_row(lhs_row, rhs_row);
 }
 
-size_t MemTable::memory_usage() {
-    return _mem_pool->mem_tracker()->consumption();
-}
+void MemTable::insert(const Tuple* tuple) {
+    bool overwritten = false;
+    if (_keys_type == KeysType::DUP_KEYS) {
+        // Will insert directly, so use memory from _table_mem_pool
+        _tuple_buf = _table_mem_pool->allocate(_schema_size);
+        ContiguousRow row(_schema, _tuple_buf);
+        _tuple_to_row(tuple, &row, _table_mem_pool.get());
+        _skip_list->Insert((char*)_tuple_buf, &overwritten);
+        DCHECK(!overwritten) << "Duplicate key model meet overwrite in SkipList";
+        return;
+    }
 
-void MemTable::insert(Tuple* tuple) {
+    _tuple_buf = _tmp_mem_pool->allocate(_schema_size);
     ContiguousRow row(_schema, _tuple_buf);
+    _tuple_to_row(tuple, &row, _table_mem_pool.get());
+
+    // TODO(lingbin): Remove redundant contain check
+    if (_skip_list->Contains((char*)_tuple_buf)) {
+        // Will aggregate, use memory from _tmp_mem_pool
+        _skip_list->Insert((char*)_tuple_buf, &overwritten);
+        DCHECK(overwritten) << "Does not meet duplicated key in SkipList";
+    } else {
+        // Will insert directly, so use memory from _table_mem_pool
+        _tuple_buf = _table_mem_pool->allocate(_schema_size);
+        ContiguousRow dst_row(_schema, _tuple_buf);
+        copy_row(&dst_row, row, _table_mem_pool.get());
+        _skip_list->Insert((char*)_tuple_buf, &overwritten);
+        DCHECK(!overwritten) << "Meet unexpected duplicated key in SkipList";
+    }
 
+    // Make MemPool to be reusable, but does not free its memory
+    _tmp_mem_pool->clear();
+}
+
+void MemTable::_tuple_to_row(const Tuple* tuple, ContiguousRow* row, MemPool* mem_pool) {
 
 Review comment:
   Seems that input mem_pool is not used.

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org