You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by zh...@apache.org on 2020/04/22 11:51:11 UTC
[incubator-doris] branch master updated: [Bug]Use
OlapTableSink::close to replace OlapTableSink::finalize method to avoid OOM
(#3363)
This is an automated email from the ASF dual-hosted git repository.
zhaoc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new a88ae53 [Bug]Use OlapTableSink::close to replace OlapTableSink::finalize method to avoid OOM (#3363)
a88ae53 is described below
commit a88ae53326848af5580c54ce1ea9f63a25ad5b7a
Author: WingC <10...@qq.com>
AuthorDate: Wed Apr 22 19:51:04 2020 +0800
[Bug]Use OlapTableSink::close to replace OlapTableSink::finalize method to avoid OOM (#3363)
This CL mainly solve the problem that when recycle `OlapTableSink`
object, GC thread will not do it immediately because the class override
the `finalize` method, and it will cause OOM.
---
fe/src/main/java/org/apache/doris/analysis/InsertStmt.java | 4 ++--
.../java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java | 2 +-
fe/src/main/java/org/apache/doris/planner/OlapTableSink.java | 2 +-
fe/src/main/java/org/apache/doris/planner/Planner.java | 2 +-
fe/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java | 2 +-
fe/src/test/java/org/apache/doris/planner/OlapTableSinkTest.java | 8 ++++----
6 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java b/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java
index f6f4cff..1089158 100644
--- a/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java
+++ b/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java
@@ -713,9 +713,9 @@ public class InsertStmt extends DdlStmt {
return dataSink;
}
- public void finalize() throws UserException {
+ public void complete() throws UserException {
if (!isExplain() && targetTable instanceof OlapTable) {
- ((OlapTableSink) dataSink).finalize();
+ ((OlapTableSink) dataSink).complete();
// add table indexes to transaction state
TransactionState txnState = Catalog.getCurrentGlobalTransactionMgr().getTransactionState(transactionId);
if (txnState == null) {
diff --git a/fe/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java b/fe/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java
index 194c2e4..ecdbd3e 100644
--- a/fe/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java
+++ b/fe/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java
@@ -128,7 +128,7 @@ public class LoadingTaskPlanner {
List<Long> partitionIds = getAllPartitionIds();
OlapTableSink olapTableSink = new OlapTableSink(table, tupleDesc, partitionIds);
olapTableSink.init(loadId, txnId, dbId, timeoutS);
- olapTableSink.finalize();
+ olapTableSink.complete();
// 3. Plan fragment
PlanFragment sinkFragment = new PlanFragment(new PlanFragmentId(0), scanNode, DataPartition.RANDOM);
diff --git a/fe/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/src/main/java/org/apache/doris/planner/OlapTableSink.java
index fa93feb..71fb975 100644
--- a/fe/src/main/java/org/apache/doris/planner/OlapTableSink.java
+++ b/fe/src/main/java/org/apache/doris/planner/OlapTableSink.java
@@ -118,7 +118,7 @@ public class OlapTableSink extends DataSink {
}
// must called after tupleDescriptor is computed
- public void finalize() throws UserException {
+ public void complete() throws UserException {
TOlapTableSink tSink = tDataSink.getOlap_table_sink();
tSink.setTable_id(dstTable.getId());
diff --git a/fe/src/main/java/org/apache/doris/planner/Planner.java b/fe/src/main/java/org/apache/doris/planner/Planner.java
index f21a111..0e82bd4 100644
--- a/fe/src/main/java/org/apache/doris/planner/Planner.java
+++ b/fe/src/main/java/org/apache/doris/planner/Planner.java
@@ -180,7 +180,7 @@ public class Planner {
InsertStmt insertStmt = (InsertStmt) statement;
rootFragment = distributedPlanner.createInsertFragment(rootFragment, insertStmt, fragments);
rootFragment.setSink(insertStmt.getDataSink());
- insertStmt.finalize();
+ insertStmt.complete();
ArrayList<Expr> exprs = ((InsertStmt) statement).getResultExprs();
List<Expr> resExprs = Expr.substituteList(
exprs, rootFragment.getPlanRoot().getOutputSmap(), analyzer, true);
diff --git a/fe/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java b/fe/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
index 639c162..5d5158f 100644
--- a/fe/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
+++ b/fe/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
@@ -114,7 +114,7 @@ public class StreamLoadPlanner {
List<Long> partitionIds = getAllPartitionIds();
OlapTableSink olapTableSink = new OlapTableSink(destTable, tupleDesc, partitionIds);
olapTableSink.init(loadId, streamLoadTask.getTxnId(), db.getId(), streamLoadTask.getTimeout());
- olapTableSink.finalize();
+ olapTableSink.complete();
// for stream load, we only need one fragment, ScanNode -> DataSink.
// OlapTableSink can dispatch data to corresponding node.
diff --git a/fe/src/test/java/org/apache/doris/planner/OlapTableSinkTest.java b/fe/src/test/java/org/apache/doris/planner/OlapTableSinkTest.java
index ac8abab..7935432 100644
--- a/fe/src/test/java/org/apache/doris/planner/OlapTableSinkTest.java
+++ b/fe/src/test/java/org/apache/doris/planner/OlapTableSinkTest.java
@@ -99,7 +99,7 @@ public class OlapTableSinkTest {
OlapTableSink sink = new OlapTableSink(dstTable, tuple, Lists.newArrayList());
sink.init(new TUniqueId(1, 2), 3, 4, 1000);
- sink.finalize();
+ sink.complete();
LOG.info("sink is {}", sink.toThrift());
LOG.info("{}", sink.getExplainString("", TExplainLevel.NORMAL));
}
@@ -130,7 +130,7 @@ public class OlapTableSinkTest {
OlapTableSink sink = new OlapTableSink(dstTable, tuple, Lists.newArrayList(p1.getId()));
sink.init(new TUniqueId(1, 2), 3, 4, 1000);
try {
- sink.finalize();
+ sink.complete();
} catch (UserException e) {
}
@@ -152,7 +152,7 @@ public class OlapTableSinkTest {
OlapTableSink sink = new OlapTableSink(dstTable, tuple, Lists.newArrayList(unknownPartId));
sink.init(new TUniqueId(1, 2), 3, 4, 1000);
- sink.finalize();
+ sink.complete();
LOG.info("sink is {}", sink.toThrift());
LOG.info("{}", sink.getExplainString("", TExplainLevel.NORMAL));
}
@@ -169,7 +169,7 @@ public class OlapTableSinkTest {
OlapTableSink sink = new OlapTableSink(dstTable, tuple, Lists.newArrayList(1L));
sink.init(new TUniqueId(1, 2), 3, 4, 1000);
- sink.finalize();
+ sink.complete();
LOG.info("sink is {}", sink.toThrift());
LOG.info("{}", sink.getExplainString("", TExplainLevel.NORMAL));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org