You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2025/10/14 03:35:49 UTC
(doris) branch branch-4.0 updated: branch-4.0: [fix](insert) do not register group commit and insert into value job to load manager #56852 (#56895)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 87d569e3234 branch-4.0: [fix](insert) do not register group commit and insert into value job to load manager #56852 (#56895)
87d569e3234 is described below
commit 87d569e3234c190fcc33ef9441d792decf6976d5
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Tue Oct 14 11:35:42 2025 +0800
branch-4.0: [fix](insert) do not register group commit and insert into value job to load manager #56852 (#56895)
Cherry-picked from #56852
Co-authored-by: hui lai <la...@selectdb.com>
---
.../commands/insert/AbstractInsertExecutor.java | 5 +++-
.../commands/insert/InsertIntoTableCommand.java | 7 +++++-
.../plans/commands/insert/OlapInsertExecutor.java | 3 ++-
.../insert_overwrite_auto_detect.groovy | 16 +-----------
.../load_p0/insert/test_insert_statistic.groovy | 29 ++++++++++++++--------
.../nereids_p0/show/test_nereids_show_load.groovy | 4 +--
6 files changed, 34 insertions(+), 30 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java
index 9b6226b417a..4448456b934 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java
@@ -95,7 +95,10 @@ public abstract class AbstractInsertExecutor {
this.ctx = ctx;
this.database = table.getDatabase();
this.insertLoadJob = new InsertLoadJob(database.getId(), labelName, jobId);
- ctx.getEnv().getLoadManager().addLoadJob(insertLoadJob);
+ // Do not add load job if job id is -1.
+ if (jobId != -1) {
+ ctx.getEnv().getLoadManager().addLoadJob(insertLoadJob);
+ }
this.coordinator = EnvFactory.getInstance().createCoordinator(
ctx, planner, ctx.getStatsErrorEstimator(), insertLoadJob.getId());
this.labelName = labelName;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java
index 6e6e499b089..c507fb4d122 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java
@@ -47,6 +47,7 @@ import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Explainable;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.algebra.InlineTable;
import org.apache.doris.nereids.trees.plans.algebra.TVFRelation;
import org.apache.doris.nereids.trees.plans.commands.Command;
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
@@ -371,10 +372,14 @@ public class InsertIntoTableCommand extends Command implements NeedAuditEncrypti
dataSink,
physicalSink,
() -> new OlapGroupCommitInsertExecutor(
- ctx, olapTable, label, planner, insertCtx, emptyInsert, groupCommitBackend, jobId
+ ctx, olapTable, label, planner, insertCtx, emptyInsert, groupCommitBackend, -1
)
);
} else {
+ // Do not register insert into value to LoadManager.
+ if (getLogicalQuery().containsType(InlineTable.class)) {
+ jobId = -1;
+ }
executorFactory = ExecutorFactory.from(
planner,
dataSink,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutor.java
index e7d19f07577..78da636fe5b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutor.java
@@ -312,7 +312,8 @@ public class OlapInsertExecutor extends AbstractInsertExecutor {
if (0 != jobId) {
etlJobType = EtlJobType.INSERT_JOB;
}
- if (!Config.enable_nereids_load) {
+ // Do not register job if job id is -1.
+ if (!Config.enable_nereids_load && jobId != -1) {
// just record for loadv2 here
ctx.getEnv().getLoadManager()
.recordFinishedLoadJob(labelName, txnId, database.getFullName(),
diff --git a/regression-test/suites/insert_overwrite_p0/insert_overwrite_auto_detect.groovy b/regression-test/suites/insert_overwrite_p0/insert_overwrite_auto_detect.groovy
index ffd3c3e642f..6c3c7e7cdd7 100644
--- a/regression-test/suites/insert_overwrite_p0/insert_overwrite_auto_detect.groovy
+++ b/regression-test/suites/insert_overwrite_p0/insert_overwrite_auto_detect.groovy
@@ -84,21 +84,7 @@ suite("test_iot_auto_detect") {
sql """ insert overwrite table list1 partition(*) with label `iot_auto_txn${uniqueID2}` values ("SHANGHAI"), ("LIST"); """
sql """ insert overwrite table list1 partition(*) with label `iot_auto_txn${uniqueID3}` values ("XXX"); """
- def max_try_milli_secs = 10000
- while(max_try_milli_secs) {
- def result = sql " show load where label like 'iot_auto_txn%' order by LoadStartTime desc "
- // the last three loads are loads upper
- if(result[0][2] == "FINISHED" && result[1][2] == "FINISHED" && result[2][2] == "FINISHED" ) {
- break
- } else {
- sleep(1000) // wait 1 second every time
- max_try_milli_secs -= 1000
- if(max_try_milli_secs <= 0) {
- log.info("result: ${result[0][2]}, ${result[1][2]}, ${result[2][2]}")
- fail()
- }
- }
- }
+ sleep(10000)
qt_sql " select * from list1 order by k0; "
diff --git a/regression-test/suites/load_p0/insert/test_insert_statistic.groovy b/regression-test/suites/load_p0/insert/test_insert_statistic.groovy
index 61df308c58e..8e55b129074 100644
--- a/regression-test/suites/load_p0/insert/test_insert_statistic.groovy
+++ b/regression-test/suites/load_p0/insert/test_insert_statistic.groovy
@@ -19,6 +19,7 @@ suite("test_insert_statistic", "p0") {
def dbName = "test_insert_statistic_db"
def insert_tbl = "test_insert_statistic_tbl"
def label = "test_insert_statistic_label"
+ sql """DROP DATABASE IF EXISTS ${dbName}"""
sql """CREATE DATABASE IF NOT EXISTS ${dbName}"""
sql """use ${dbName}"""
@@ -42,10 +43,18 @@ suite("test_insert_statistic", "p0") {
INSERT INTO ${insert_tbl}_1 values(1, 1, 1, 1)
"""
def result = sql "SHOW LOAD FROM ${dbName}"
- logger.info("JobDetails: " + result[0][14])
- def json = parseJson(result[0][14])
- assertEquals(json.ScannedRows, 1)
- assertTrue(json.LoadBytes > 0)
+ log.info("result size: " + result.size())
+ assertEquals(result.size(), 0)
+
+ // group commit
+ sql """ set group_commit = sync_mode; """
+ sql """
+ INSERT INTO ${insert_tbl}_1 values(1, 1, 1, 1)
+ """
+ sql """ set group_commit = off_mode; """
+ result = sql "SHOW LOAD FROM ${dbName}"
+ log.info("result size: " + result.size())
+ assertEquals(result.size(), 0)
// insert into select
sql """ DROP TABLE IF EXISTS ${insert_tbl}_2"""
@@ -67,9 +76,9 @@ suite("test_insert_statistic", "p0") {
INSERT INTO ${insert_tbl}_2 select * from ${insert_tbl}_1
"""
result = sql "SHOW LOAD FROM ${dbName}"
- logger.info("JobDetails: " + result[1][14])
- json = parseJson(result[1][14])
- assertEquals(json.ScannedRows, 1)
+ logger.info("JobDetails: " + result[0][14])
+ def json = parseJson(result[0][14])
+ assertEquals(json.ScannedRows, 2)
assertTrue(json.LoadBytes > 0)
// insert into s3 tvf
@@ -104,8 +113,8 @@ suite("test_insert_statistic", "p0") {
);
"""
result = sql "SHOW LOAD FROM ${dbName}"
- logger.info("JobDetails: " + result[2][14])
- json = parseJson(result[2][14])
- assertEquals(json.ScannedRows, 1)
+ logger.info("JobDetails: " + result[1][14])
+ json = parseJson(result[1][14])
+ assertEquals(json.ScannedRows, 2)
assertTrue(json.LoadBytes > 0)
}
\ No newline at end of file
diff --git a/regression-test/suites/nereids_p0/show/test_nereids_show_load.groovy b/regression-test/suites/nereids_p0/show/test_nereids_show_load.groovy
index afb928f2ab4..73a273b8d99 100644
--- a/regression-test/suites/nereids_p0/show/test_nereids_show_load.groovy
+++ b/regression-test/suites/nereids_p0/show/test_nereids_show_load.groovy
@@ -31,8 +31,8 @@ suite("test_nereids_show_load") {
PROPERTIES ("replication_allocation" = "tag.location.default: 1");
"""
- sql """ insert into test_nereids_show_load with label `test_nereids_show_load_label_xyz` values (1,1); """
- sql """ insert into test_nereids_show_load with label `new_test_nereids_show_load_label_xyz` values (2,2); """
+ sql """ insert into test_nereids_show_load with label `test_nereids_show_load_label_xyz` select 1,1; """
+ sql """ insert into test_nereids_show_load with label `new_test_nereids_show_load_label_xyz` select 2,2; """
checkNereidsExecute("SHOW LOAD")
checkNereidsExecute("SHOW LOAD FROM test_nereids_show_load_db")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org