You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2025/11/08 13:23:34 UTC

(doris) branch branch-4.0 updated: branch-4.0: [fix](export) cancel job before adding task (#57488) (#57792)

This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-4.0 by this push:
     new 3044d79bba3 branch-4.0: [fix](export) cancel job before adding task (#57488) (#57792)
3044d79bba3 is described below

commit 3044d79bba3a5ac23192fae6027b44b32542a995
Author: Mingyu Chen (Rayner) <mo...@163.com>
AuthorDate: Sat Nov 8 21:23:27 2025 +0800

    branch-4.0: [fix](export) cancel job before adding task (#57488) (#57792)
    
    bp #57488
---
 .../org/apache/doris/analysis/OutFileClause.java   |  9 ++-
 .../main/java/org/apache/doris/load/ExportMgr.java | 18 +++---
 .../trees/plans/commands/ExportCommand.java        | 22 ++++++-
 .../export_p0/test_export_delete_disallow.groovy   | 75 ++++++++++++++++++++++
 4 files changed, 109 insertions(+), 15 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
index cab36a7bb62..78bc458b76f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
@@ -525,8 +525,12 @@ public class OutFileClause {
         }
 
         if (copiedProps.containsKey(PROP_DELETE_EXISTING_FILES)) {
-            deleteExistingFiles = Boolean.parseBoolean(copiedProps.get(PROP_DELETE_EXISTING_FILES))
-                    & Config.enable_delete_existing_files;
+            deleteExistingFiles = Boolean.parseBoolean(copiedProps.get(PROP_DELETE_EXISTING_FILES));
+            if (deleteExistingFiles && !Config.enable_delete_existing_files) {
+                throw new AnalysisException("Deleting existing files is not allowed."
+                        + " To enable this feature, you need to add `enable_delete_existing_files=true`"
+                        + " in fe.conf");
+            }
             copiedProps.remove(PROP_DELETE_EXISTING_FILES);
         }
 
@@ -757,3 +761,4 @@ public class OutFileClause {
     }
 }
 
+
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
index 25eef76aeac..8de5a1c95c6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
@@ -111,17 +111,14 @@ public class ExportMgr {
         } finally {
             writeUnlock();
         }
-        // delete existing files
-        if (Config.enable_delete_existing_files && Boolean.parseBoolean(job.getDeleteExistingFiles())) {
-            if (job.getBrokerDesc() == null) {
-                throw new AnalysisException("Local file system does not support delete existing files");
-            }
-            String fullPath = job.getExportPath();
-            BrokerUtil.deleteDirectoryWithFileSystem(fullPath.substring(0, fullPath.lastIndexOf('/') + 1),
-                    job.getBrokerDesc());
-        }
-        // ATTN: Must add task after edit log, otherwise the job may finish before adding job.
         try {
+            // delete existing files
+            if (Boolean.parseBoolean(job.getDeleteExistingFiles())) {
+                String fullPath = job.getExportPath();
+                BrokerUtil.deleteDirectoryWithFileSystem(fullPath.substring(0, fullPath.lastIndexOf('/') + 1),
+                        job.getBrokerDesc());
+            }
+            // ATTN: Must add task after edit log, otherwise the job may finish before adding job.
             for (int i = 0; i < job.getCopiedTaskExecutors().size(); i++) {
                 Env.getCurrentEnv().getTransientTaskManager().addMemoryTask(job.getCopiedTaskExecutors().get(i));
             }
@@ -557,3 +554,4 @@ public class ExportMgr {
         return size;
     }
 }
+
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
index 1de29d7c1f0..e1ee813224c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
@@ -22,6 +22,7 @@ import org.apache.doris.analysis.OutFileClause;
 import org.apache.doris.analysis.Separator;
 import org.apache.doris.analysis.StmtType;
 import org.apache.doris.analysis.StorageBackend;
+import org.apache.doris.analysis.StorageBackend.StorageType;
 import org.apache.doris.analysis.TableName;
 import org.apache.doris.catalog.BrokerMgr;
 import org.apache.doris.catalog.DatabaseIf;
@@ -305,9 +306,6 @@ public class ExportCommand extends Command implements NeedAuditEncryption, Forwa
 
         // set max_file_size
         exportJob.setMaxFileSize(fileProperties.getOrDefault(OutFileClause.PROP_MAX_FILE_SIZE, ""));
-        // set delete_existing_files
-        exportJob.setDeleteExistingFiles(fileProperties.getOrDefault(
-                OutFileClause.PROP_DELETE_EXISTING_FILES, ""));
 
         // null means not specified
         // "" means user specified zero columns
@@ -322,6 +320,23 @@ public class ExportCommand extends Command implements NeedAuditEncryption, Forwa
         // set broker desc
         exportJob.setBrokerDesc(this.brokerDesc.get());
 
+        // set delete_existing_files
+        exportJob.setDeleteExistingFiles(fileProperties.getOrDefault(
+                OutFileClause.PROP_DELETE_EXISTING_FILES, ""));
+
+        if (!Config.enable_delete_existing_files && exportJob.getDeleteExistingFiles().equalsIgnoreCase("true")) {
+            throw new AnalysisException(("Deleting existing files is not allowed."
+                    + " To enable this feature, you need to add `enable_delete_existing_files=true`"
+                    + " in fe.conf"));
+        }
+
+        if (exportJob.getDeleteExistingFiles().equalsIgnoreCase("true")
+                && (exportJob.getBrokerDesc() == null
+                || exportJob.getBrokerDesc().storageType() == StorageType.LOCAL)) {
+            throw new org.apache.doris.common.AnalysisException(
+                    "Local file system does not support delete existing files");
+        }
+
         // set sessions
         exportJob.setQualifiedUser(ctx.getQualifiedUser());
         exportJob.setUserIdentity(ctx.getCurrentUserIdentity());
@@ -400,3 +415,4 @@ public class ExportCommand extends Command implements NeedAuditEncryption, Forwa
         return true;
     }
 }
+
diff --git a/regression-test/suites/export_p0/test_export_delete_disallow.groovy b/regression-test/suites/export_p0/test_export_delete_disallow.groovy
new file mode 100644
index 00000000000..f5f508afb52
--- /dev/null
+++ b/regression-test/suites/export_p0/test_export_delete_disallow.groovy
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("test_export_delete_disallow", "p0") {
+    def db = "regression_test_export_p0"
+    
+    // check whether the FE config 'enable_outfile_to_local' is true
+    StringBuilder strBuilder = new StringBuilder()
+    strBuilder.append("curl --location-trusted -u " + context.config.jdbcUser + ":" + context.config.jdbcPassword)
+    strBuilder.append(" http://" + context.config.feHttpAddress + "/rest/v1/config/fe")
+
+    String command = strBuilder.toString()
+    def process = command.toString().execute()
+    def code = process.waitFor()
+    def err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream())));
+    def out = process.getText()
+    logger.info("Request FE Config: code=" + code + ", out=" + out + ", err=" + err)
+    assertEquals(code, 0)
+    def response = parseJson(out.trim())
+    assertEquals(response.code, 0)
+    assertEquals(response.msg, "success")
+    def configJson = response.data.rows
+    boolean enableOutfileToLocal = false
+    for (Object conf: configJson) {
+        assert conf instanceof Map
+        if (((Map<String, String>) conf).get("Name").toLowerCase() == "enable_outfile_to_local") {
+            enableOutfileToLocal = ((Map<String, String>) conf).get("Value").toLowerCase() == "true"
+        }
+    }
+    if (!enableOutfileToLocal) {
+        logger.warn("Please set enable_outfile_to_local to true to run test_outfile")
+        return
+    }
+    
+    // create table and insert
+    sql """ DROP TABLE IF EXISTS test_export_delete_disallow_tbl"""
+    sql """
+    CREATE TABLE IF NOT EXISTS test_export_delete_disallow_tbl (
+        `id` int(11) NULL,
+        `Name` string NULL
+        )
+        PROPERTIES("replication_num" = "1");
+    """
+    sql """insert into test_export_delete_disallow_tbl values(1, "abc");"""
+    
+    test {
+         sql """select * from test_export_delete_disallow_tbl limit 10 INTO OUTFILE "file:///tmp/" properties("delete_existing_files" = "true");"""
+         exception """Deleting existing files is not allowed"""
+    }
+
+    test {
+        sql """EXPORT TABLE test_export_delete_disallow_tbl TO "file:///tmp" PROPERTIES ("delete_existing_files" = "true");"""
+        exception """Deleting existing files is not allowe"""
+    }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org