You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2025/11/08 13:23:34 UTC
(doris) branch branch-4.0 updated: branch-4.0: [fix](export) cancel job before adding task (#57488) (#57792)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 3044d79bba3 branch-4.0: [fix](export) cancel job before adding task (#57488) (#57792)
3044d79bba3 is described below
commit 3044d79bba3a5ac23192fae6027b44b32542a995
Author: Mingyu Chen (Rayner) <mo...@163.com>
AuthorDate: Sat Nov 8 21:23:27 2025 +0800
branch-4.0: [fix](export) cancel job before adding task (#57488) (#57792)
bp #57488
---
.../org/apache/doris/analysis/OutFileClause.java | 9 ++-
.../main/java/org/apache/doris/load/ExportMgr.java | 18 +++---
.../trees/plans/commands/ExportCommand.java | 22 ++++++-
.../export_p0/test_export_delete_disallow.groovy | 75 ++++++++++++++++++++++
4 files changed, 109 insertions(+), 15 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
index cab36a7bb62..78bc458b76f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
@@ -525,8 +525,12 @@ public class OutFileClause {
}
if (copiedProps.containsKey(PROP_DELETE_EXISTING_FILES)) {
- deleteExistingFiles = Boolean.parseBoolean(copiedProps.get(PROP_DELETE_EXISTING_FILES))
- & Config.enable_delete_existing_files;
+ deleteExistingFiles = Boolean.parseBoolean(copiedProps.get(PROP_DELETE_EXISTING_FILES));
+ if (deleteExistingFiles && !Config.enable_delete_existing_files) {
+ throw new AnalysisException("Deleting existing files is not allowed."
+ + " To enable this feature, you need to add `enable_delete_existing_files=true`"
+ + " in fe.conf");
+ }
copiedProps.remove(PROP_DELETE_EXISTING_FILES);
}
@@ -757,3 +761,4 @@ public class OutFileClause {
}
}
+
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
index 25eef76aeac..8de5a1c95c6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
@@ -111,17 +111,14 @@ public class ExportMgr {
} finally {
writeUnlock();
}
- // delete existing files
- if (Config.enable_delete_existing_files && Boolean.parseBoolean(job.getDeleteExistingFiles())) {
- if (job.getBrokerDesc() == null) {
- throw new AnalysisException("Local file system does not support delete existing files");
- }
- String fullPath = job.getExportPath();
- BrokerUtil.deleteDirectoryWithFileSystem(fullPath.substring(0, fullPath.lastIndexOf('/') + 1),
- job.getBrokerDesc());
- }
- // ATTN: Must add task after edit log, otherwise the job may finish before adding job.
try {
+ // delete existing files
+ if (Boolean.parseBoolean(job.getDeleteExistingFiles())) {
+ String fullPath = job.getExportPath();
+ BrokerUtil.deleteDirectoryWithFileSystem(fullPath.substring(0, fullPath.lastIndexOf('/') + 1),
+ job.getBrokerDesc());
+ }
+ // ATTN: Must add task after edit log, otherwise the job may finish before adding job.
for (int i = 0; i < job.getCopiedTaskExecutors().size(); i++) {
Env.getCurrentEnv().getTransientTaskManager().addMemoryTask(job.getCopiedTaskExecutors().get(i));
}
@@ -557,3 +554,4 @@ public class ExportMgr {
return size;
}
}
+
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
index 1de29d7c1f0..e1ee813224c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
@@ -22,6 +22,7 @@ import org.apache.doris.analysis.OutFileClause;
import org.apache.doris.analysis.Separator;
import org.apache.doris.analysis.StmtType;
import org.apache.doris.analysis.StorageBackend;
+import org.apache.doris.analysis.StorageBackend.StorageType;
import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.BrokerMgr;
import org.apache.doris.catalog.DatabaseIf;
@@ -305,9 +306,6 @@ public class ExportCommand extends Command implements NeedAuditEncryption, Forwa
// set max_file_size
exportJob.setMaxFileSize(fileProperties.getOrDefault(OutFileClause.PROP_MAX_FILE_SIZE, ""));
- // set delete_existing_files
- exportJob.setDeleteExistingFiles(fileProperties.getOrDefault(
- OutFileClause.PROP_DELETE_EXISTING_FILES, ""));
// null means not specified
// "" means user specified zero columns
@@ -322,6 +320,23 @@ public class ExportCommand extends Command implements NeedAuditEncryption, Forwa
// set broker desc
exportJob.setBrokerDesc(this.brokerDesc.get());
+ // set delete_existing_files
+ exportJob.setDeleteExistingFiles(fileProperties.getOrDefault(
+ OutFileClause.PROP_DELETE_EXISTING_FILES, ""));
+
+ if (!Config.enable_delete_existing_files && exportJob.getDeleteExistingFiles().equalsIgnoreCase("true")) {
+ throw new AnalysisException(("Deleting existing files is not allowed."
+ + " To enable this feature, you need to add `enable_delete_existing_files=true`"
+ + " in fe.conf"));
+ }
+
+ if (exportJob.getDeleteExistingFiles().equalsIgnoreCase("true")
+ && (exportJob.getBrokerDesc() == null
+ || exportJob.getBrokerDesc().storageType() == StorageType.LOCAL)) {
+ throw new org.apache.doris.common.AnalysisException(
+ "Local file system does not support delete existing files");
+ }
+
// set sessions
exportJob.setQualifiedUser(ctx.getQualifiedUser());
exportJob.setUserIdentity(ctx.getCurrentUserIdentity());
@@ -400,3 +415,4 @@ public class ExportCommand extends Command implements NeedAuditEncryption, Forwa
return true;
}
}
+
diff --git a/regression-test/suites/export_p0/test_export_delete_disallow.groovy b/regression-test/suites/export_p0/test_export_delete_disallow.groovy
new file mode 100644
index 00000000000..f5f508afb52
--- /dev/null
+++ b/regression-test/suites/export_p0/test_export_delete_disallow.groovy
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("test_export_delete_disallow", "p0") {
+ def db = "regression_test_export_p0"
+
+ // check whether the FE config 'enable_outfile_to_local' is true
+ StringBuilder strBuilder = new StringBuilder()
+ strBuilder.append("curl --location-trusted -u " + context.config.jdbcUser + ":" + context.config.jdbcPassword)
+ strBuilder.append(" http://" + context.config.feHttpAddress + "/rest/v1/config/fe")
+
+ String command = strBuilder.toString()
+ def process = command.toString().execute()
+ def code = process.waitFor()
+ def err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream())));
+ def out = process.getText()
+ logger.info("Request FE Config: code=" + code + ", out=" + out + ", err=" + err)
+ assertEquals(code, 0)
+ def response = parseJson(out.trim())
+ assertEquals(response.code, 0)
+ assertEquals(response.msg, "success")
+ def configJson = response.data.rows
+ boolean enableOutfileToLocal = false
+ for (Object conf: configJson) {
+ assert conf instanceof Map
+ if (((Map<String, String>) conf).get("Name").toLowerCase() == "enable_outfile_to_local") {
+ enableOutfileToLocal = ((Map<String, String>) conf).get("Value").toLowerCase() == "true"
+ }
+ }
+ if (!enableOutfileToLocal) {
+ logger.warn("Please set enable_outfile_to_local to true to run test_outfile")
+ return
+ }
+
+ // create table and insert
+ sql """ DROP TABLE IF EXISTS test_export_delete_disallow_tbl"""
+ sql """
+ CREATE TABLE IF NOT EXISTS test_export_delete_disallow_tbl (
+ `id` int(11) NULL,
+ `Name` string NULL
+ )
+ PROPERTIES("replication_num" = "1");
+ """
+ sql """insert into test_export_delete_disallow_tbl values(1, "abc");"""
+
+ test {
+ sql """select * from test_export_delete_disallow_tbl limit 10 INTO OUTFILE "file:///tmp/" properties("delete_existing_files" = "true");"""
+ exception """Deleting existing files is not allowed"""
+ }
+
+ test {
+ sql """EXPORT TABLE test_export_delete_disallow_tbl TO "file:///tmp" PROPERTIES ("delete_existing_files" = "true");"""
+ exception """Deleting existing files is not allowe"""
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org