You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/07/13 03:17:35 UTC

[doris] branch master updated: [Fix](Planner) fix limit execute before sort in show export job (#21663)

This is an automated email from the ASF dual-hosted git repository.

morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new f863c653e2 [Fix](Planner) fix limit execute before sort in show export job (#21663)
f863c653e2 is described below

commit f863c653e26dc3d32ee28ae00352152123b5ae90
Author: LiBinfeng <46...@users.noreply.github.com>
AuthorDate: Thu Jul 13 11:17:28 2023 +0800

    [Fix](Planner) fix limit execute before sort in show export job (#21663)
    
    Problem:
    When doing show export jobs, limit would execute before sort before changed. So the result would not be expected because limit always cut results first and we can not get what we want.
    
    Example:
    we having export job1 and job2 with JobId1 > JobId2. We want to get job with JobId1
    show export from db order by JobId desc limit 1;
    We do limit 1 first, so we would probably get Job2 because JobId assigned from small to large
    
    Solve:
    We can not cut results first if we have order by clause. And cut result set after sorting
---
 .../main/java/org/apache/doris/load/ExportMgr.java |  6 ++-
 .../suites/export_p0/test_export_basic.groovy      | 43 ++++++++++++++++++++++
 2 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
index a81144138e..0e0b26f273 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
@@ -307,7 +307,7 @@ public class ExportMgr extends MasterDaemon {
                     exportJobInfos.add(composeExportJobInfo(job));
                 }
 
-                if (++counter >= resultNum) {
+                if (++counter >= resultNum && orderByPairs == null) {
                     break;
                 }
             }
@@ -327,8 +327,12 @@ public class ExportMgr extends MasterDaemon {
         Collections.sort(exportJobInfos, comparator);
 
         List<List<String>> results = Lists.newArrayList();
+        int counter = 0;
         for (List<Comparable> list : exportJobInfos) {
             results.add(list.stream().map(e -> e.toString()).collect(Collectors.toList()));
+            if (++counter >= resultNum) {
+                break;
+            }
         }
 
         return results;
diff --git a/regression-test/suites/export_p0/test_export_basic.groovy b/regression-test/suites/export_p0/test_export_basic.groovy
index 07c3168c5d..95805db491 100644
--- a/regression-test/suites/export_p0/test_export_basic.groovy
+++ b/regression-test/suites/export_p0/test_export_basic.groovy
@@ -386,4 +386,47 @@ suite("test_export_basic", "p0") {
         try_sql("DROP TABLE IF EXISTS ${table_load_name}")
         delete_files.call("${outFilePath}")
     }
+
+    // 5. test order by and limit clause
+    uuid1 = UUID.randomUUID().toString()
+    outFilePath = """${outfile_path_prefix}_${uuid1}"""
+    label1 = "label_${uuid1}"
+    uuid2 = UUID.randomUUID().toString()
+    label2 = "label_${uuid2}"
+    try {
+        // check export path
+        check_path_exists.call("${outFilePath}")
+
+        // exec export
+        sql """
+            EXPORT TABLE ${table_export_name} PARTITION (less_than_20)
+            TO "file://${outFilePath}/"
+            PROPERTIES(
+                "label" = "${label1}",
+                "format" = "csv",
+                "column_separator"=","
+            );
+        """
+        sql """
+            EXPORT TABLE ${table_export_name} PARTITION (between_20_70)
+            TO "file://${outFilePath}/"
+            PROPERTIES(
+                "label" = "${label2}",
+                "format" = "csv",
+                "column_separator"=","
+            );
+        """
+        waiting_export.call(label1)
+        waiting_export.call(label2)
+
+        // check file amounts
+        check_file_amounts.call("${outFilePath}", 2)
+
+        // check show export correctness
+        def res = sql """ show export where STATE = "FINISHED" order by JobId desc limit 2"""
+        assertTrue(res[0][0] > res[1][0])
+
+    } finally {
+        delete_files.call("${outFilePath}")
+    }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org