[incubator-doris] branch master updated: [Bug] fix finished load jobs cost too much heap (#4993)

[li...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

lingmiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git


The following commit(s) were added to refs/heads/master by this push:
     new b4c1eab  [Bug] fix finished load jobs cost too much heap  (#4993)
b4c1eab is described below

commit b4c1eabe3fd0b986ef876b5a99020bebc42a4aba
Author: qiye <ji...@gmail.com>
AuthorDate: Wed Dec 2 17:11:27 2020 +0800

    [Bug] fix finished load jobs cost too much heap  (#4993)
    
    Since the plan is retained in the task, if the task is not cleaned up, the memory usage will be too large caused Memory leak or OOM.
    When load job finished, there is no need to hold the tasks which are the biggest memory consumers.
    Fixed #4992
---
 fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java  | 2 ++
 .../src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java     | 6 +++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java
index b3d3b1b..8b8ff0d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java
@@ -659,6 +659,8 @@ public abstract class LoadJob extends AbstractTxnStateChangeCallback implements
         if (MetricRepo.isInit) {
             MetricRepo.COUNTER_LOAD_FINISHED.increase(1L);
         }
+        // when load job finished, there is no need to hold the tasks which are the biggest memory consumers.
+        idToTasks.clear();
     }
 
     protected boolean checkDataQuality() {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java
index f324b76..ad981d7 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java
@@ -40,13 +40,13 @@ import org.apache.doris.transaction.TransactionState;
 import com.google.common.collect.Maps;
 
 import org.junit.Assert;
-import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.util.Map;
 
 import mockit.Expectations;
+import mockit.Injectable;
 import mockit.Mocked;
 
 public class LoadJobTest {
@@ -187,18 +187,22 @@ public class LoadJobTest {
 
     @Test
     public void testUpdateStateToFinished(@Mocked MetricRepo metricRepo,
+                                          @Injectable LoadTask loadTask1,
             @Mocked LongCounterMetric longCounterMetric) {
         
         MetricRepo.COUNTER_LOAD_FINISHED = longCounterMetric;
         LoadJob loadJob = new BrokerLoadJob();
+        loadJob.idToTasks.put(1L, loadTask1);
         
         // TxnStateCallbackFactory factory = Catalog.getCurrentCatalog().getGlobalTransactionMgr().getCallbackFactory();
         Catalog catalog = Catalog.getCurrentCatalog();
         GlobalTransactionMgr mgr = new GlobalTransactionMgr(catalog);
         Deencapsulation.setField(catalog, "globalTransactionMgr", mgr);
+        Assert.assertEquals(1, loadJob.idToTasks.size());
         loadJob.updateState(JobState.FINISHED);
         Assert.assertEquals(JobState.FINISHED, loadJob.getState());
         Assert.assertNotEquals(-1, (long) Deencapsulation.getField(loadJob, "finishTimestamp"));
         Assert.assertEquals(100, (int)Deencapsulation.getField(loadJob, "progress"));
+        Assert.assertEquals(0, loadJob.idToTasks.size());
     }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org