[incubator-doris] branch master updated: [Bug] Fix schema change fail as memory allocation on row block sorting (#6932)

[mo...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 2d10300  [Bug] Fix schema change fail as memory allocation on row block sorting (#6932)
2d10300 is described below

commit 2d1030054734435f05476144d81ee0dd9e1a0864
Author: GoGoWen <82...@users.noreply.github.com>
AuthorDate: Tue Nov 2 16:33:38 2021 +0800

    [Bug] Fix schema change fail as memory allocation on row block sorting (#6932)
    
    schema change fail as memory allocation fail on row block sorting.
    however, it should do internal sorting first before schema change fail
    as memory allocation fail on row block sorting in case there are enough
    memory after internal sorting.
---
 be/src/olap/schema_change.cpp | 22 ++++++++++++++++++++++
 be/src/olap/schema_change.h   |  1 +
 2 files changed, 23 insertions(+)

diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index 8a1eb34..8b3fc49 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -56,6 +56,9 @@ class RowBlockSorter {
 public:
     explicit RowBlockSorter(RowBlockAllocator* allocator);
     virtual ~RowBlockSorter();
+    size_t num_rows() { 
+        return _swap_row_block->capacity(); 
+    }
 
     bool sort(RowBlock** row_block);
 
@@ -800,6 +803,15 @@ void RowBlockAllocator::release(RowBlock* row_block) {
     delete row_block;
 }
 
+bool RowBlockAllocator::is_memory_enough_for_sorting(size_t num_rows, size_t allocated_rows){
+    if (num_rows <= allocated_rows) {
+        return true;
+    }
+    size_t row_block_size = _row_len * (num_rows - allocated_rows);
+    return _mem_tracker->consumption() + row_block_size < _memory_limitation;
+}
+
+
 RowBlockMerger::RowBlockMerger(TabletSharedPtr tablet) : _tablet(tablet) {}
 
 RowBlockMerger::~RowBlockMerger() {}
@@ -1184,6 +1196,16 @@ OLAPStatus SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader,
                                                            true)) {
             LOG(WARNING) << "failed to allocate RowBlock.";
             return OLAP_ERR_INPUT_PARAMETER_ERROR;
+        } else {
+            // do memory check for sorting, in case schema change task fail at row block sorting because of 
+            // not doing internal sorting first
+            if (!_row_block_allocator->is_memory_enough_for_sorting(ref_row_block->row_block_info().row_num,
+                                                                row_block_sorter.num_rows())) {
+                if (new_row_block != nullptr) {
+                    _row_block_allocator->release(new_row_block);
+                    new_row_block = nullptr;
+                }
+            }
         }
 
         if (new_row_block == nullptr) {
diff --git a/be/src/olap/schema_change.h b/be/src/olap/schema_change.h
index 73f784a..e0d42e8 100644
--- a/be/src/olap/schema_change.h
+++ b/be/src/olap/schema_change.h
@@ -81,6 +81,7 @@ public:
 
     OLAPStatus allocate(RowBlock** row_block, size_t num_rows, bool null_supported);
     void release(RowBlock* row_block);
+    bool is_memory_enough_for_sorting(size_t num_rows, size_t allocated_rows);
 
 private:
     const TabletSchema& _tablet_schema;

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org