You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/03/24 00:07:22 UTC
(doris) branch branch-2.1 updated (8b349155182 -> 5c3ca0fbc2b)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a change to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
from 8b349155182 [Fix](compress) Fix occasional crushes when serializing blocks (#32672)
new 735f5213c10 Add log for case to help debug. (#32675)
new 8b960beaec8 [refactor](nereids)unify outputTupleDesc and projection (#32093)
new c223d9e7d00 [Fix](Test)Reduce the Sleep time and ensure that the corresponding thread resources are released after the case is executed. (#32665)
new a55b5ea9caa [bug](udaf) fix memory leak in the java udaf (#32630)
new 4ebbabf15e8 [test](fuzzy) test fuzzy in BE (#31607)
new 3d14d9e379d [fix](Nereids) fix bind having aggregate failed again (#32687)
new f62cdecc08b [fix](Nereids) do not push down topn-filter through right/full outer join if the first orderkey is nulls first (#32633)
new ac8ba43d8de [fix](multi-catalog)resolve hive meta store compatibility for different version issues (#32551)
new bd2e7d0ec88 [feature](hive)support insert overwrite (#32610)
new 33e40345b6c [Improve](build-script)Add be-extension-ignore to ignore avro-scanner when building release packages (#32699)
new 1d4e5a1c58f [enhance](auth)enable col auth (#32659)
new 5077f7dd9dd [conf](mysql) opt mysql network timeout to 600s #32545
new 5c3fc818bb8 [Fix](Export) fix variable name of code #32637
new b9788e5e37a [fix] (Nereids) fix date function rewrite on datetimev1 column bug (#32569)
new 5c3ca0fbc2b [fix](move-memtable) fix load timeout caused by lost wakeup (#32720)
The 15 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/common/config.cpp | 47 ++--
be/src/common/config.h | 24 --
.../aggregate_function_java_udaf.h | 35 ++-
be/src/vec/sink/load_stream_stub.cpp | 25 +-
build-for-release.sh | 2 +-
build.sh | 36 +++
.../main/java/org/apache/doris/common/Config.java | 5 -
.../java/org/apache/doris/analysis/ExportStmt.java | 2 +-
.../java/org/apache/doris/analysis/GrantStmt.java | 3 +-
.../apache/doris/datasource/hive/HMSCommitter.java | 96 ++++++-
.../main/java/org/apache/doris/fs/FileSystem.java | 6 +
.../apache/doris/fs/remote/dfs/DFSFileSystem.java | 40 +--
.../apache/doris/job/extensions/mtmv/MTMVTask.java | 1 +
.../main/java/org/apache/doris/load/ExportJob.java | 2 +-
.../glue/translator/PhysicalPlanTranslator.java | 14 +-
.../doris/nereids/processor/post/TopNScanOpt.java | 16 +-
.../nereids/rules/analysis/BindExpression.java | 37 +--
.../trees/expressions/literal/DateLiteral.java | 4 +-
.../trees/plans/commands/ExportCommand.java | 2 +-
.../commands/insert/AbstractInsertExecutor.java | 1 +
.../org/apache/doris/planner/HashJoinNode.java | 34 +--
.../org/apache/doris/planner/JoinNodeBase.java | 78 +++---
.../apache/doris/planner/NestedLoopJoinNode.java | 30 +--
.../java/org/apache/doris/planner/PlanNode.java | 16 +-
.../java/org/apache/doris/qe/SessionVariable.java | 4 +-
.../hadoop/hive/metastore/HiveMetaStoreClient.java | 98 ++++---
.../doris/datasource/hive/HmsCommitTest.java | 56 +++-
.../org/apache/doris/mysql/privilege/AuthTest.java | 2 -
.../datetimev1/test_datetimev1_date_function.out | 32 +++
.../data/nereids_syntax_p0/bind_priority.out | 9 +
.../test_datetimev1_date_function.groovy | 124 +++++++++
.../suites/job_p0/test_base_insert_job.groovy | 286 +++++++++++++++++++++
.../suites/nereids_syntax_p0/bind_priority.groovy | 21 ++
.../suites/nereids_tpch_p0/tpch/topn-filter.groovy | 15 ++
.../table_function/explode_json_array.groovy | 32 ++-
.../suites/statistics/analyze_stats.groovy | 23 ++
36 files changed, 954 insertions(+), 304 deletions(-)
create mode 100644 regression-test/data/datatype_p0/datetimev1/test_datetimev1_date_function.out
create mode 100644 regression-test/suites/datatype_p0/datetimev1/test_datetimev1_date_function.groovy
create mode 100644 regression-test/suites/job_p0/test_base_insert_job.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 09/15: [feature](hive)support insert overwrite (#32610)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit bd2e7d0ec881b6724cde7b31a4a7da4e6dcc1883
Author: wuwenchi <wu...@hotmail.com>
AuthorDate: Fri Mar 22 21:32:20 2024 +0800
[feature](hive)support insert overwrite (#32610)
support insert overwrite for unpartitioned table and partitioned table.
issue: #31442
---
.../apache/doris/datasource/hive/HMSCommitter.java | 96 ++++++++++++++++++++--
.../main/java/org/apache/doris/fs/FileSystem.java | 6 ++
.../apache/doris/fs/remote/dfs/DFSFileSystem.java | 40 +++++----
.../doris/datasource/hive/HmsCommitTest.java | 56 ++++++++++---
4 files changed, 166 insertions(+), 32 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCommitter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCommitter.java
index 64abb985fcf..af26f36d6b9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCommitter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCommitter.java
@@ -80,6 +80,8 @@ public class HMSCommitter {
private final Queue<DirectoryCleanUpTask> directoryCleanUpTasksForAbort = new ConcurrentLinkedQueue<>();
// when aborted, we need restore directory
private final List<RenameDirectoryTask> renameDirectoryTasksForAbort = new ArrayList<>();
+ // when finished, we need clear some directories
+ private final List<String> clearDirsForFinish = new ArrayList<>();
Executor fileSystemExecutor = Executors.newFixedThreadPool(16);
public HMSCommitter(HiveMetadataOps hiveOps, RemoteFileSystem fs, Table table) {
@@ -105,6 +107,8 @@ public class HMSCommitter {
t.addSuppressed(new Exception("Failed to roll back after commit failure", e));
}
throw t;
+ } finally {
+ runClearPathsForFinish();
}
}
@@ -250,7 +254,38 @@ public class HMSCommitter {
}
public void prepareOverwriteTable(THivePartitionUpdate pu, HivePartitionStatistics ps) {
-
+ String targetPath = pu.getLocation().getTargetPath();
+ String writePath = pu.getLocation().getWritePath();
+ if (!targetPath.equals(writePath)) {
+ Path path = new Path(targetPath);
+ String oldTablePath = new Path(path.getParent(), "_temp_" + path.getName()).toString();
+ Status status = fs.renameDir(
+ targetPath,
+ oldTablePath,
+ () -> renameDirectoryTasksForAbort.add(new RenameDirectoryTask(oldTablePath, targetPath)));
+ if (!status.ok()) {
+ throw new RuntimeException(
+ "Error to rename dir from " + targetPath + " to " + oldTablePath + status.getErrMsg());
+ }
+ clearDirsForFinish.add(oldTablePath);
+
+ status = fs.renameDir(
+ writePath,
+ targetPath,
+ () -> directoryCleanUpTasksForAbort.add(new DirectoryCleanUpTask(targetPath, true)));
+ if (!status.ok()) {
+ throw new RuntimeException(
+ "Error to rename dir from " + writePath + " to " + targetPath + ":" + status.getErrMsg());
+ }
+ }
+ updateStatisticsTasks.add(
+ new UpdateStatisticsTask(
+ table.getDbName(),
+ table.getTableName(),
+ Optional.empty(),
+ ps,
+ false
+ ));
}
public void prepareCreateNewPartition(THivePartitionUpdate pu, HivePartitionStatistics ps) {
@@ -335,7 +370,38 @@ public class HMSCommitter {
public void prepareOverwritePartition(THivePartitionUpdate pu, HivePartitionStatistics ps) {
+ String targetPath = pu.getLocation().getTargetPath();
+ String writePath = pu.getLocation().getWritePath();
+ if (!targetPath.equals(writePath)) {
+ Path path = new Path(targetPath);
+ String oldPartitionPath = new Path(path.getParent(), "_temp_" + path.getName()).toString();
+ Status status = fs.renameDir(
+ targetPath,
+ oldPartitionPath,
+ () -> renameDirectoryTasksForAbort.add(new RenameDirectoryTask(oldPartitionPath, targetPath)));
+ if (!status.ok()) {
+ throw new RuntimeException(
+ "Error to rename dir from " + targetPath + " to " + oldPartitionPath + ":" + status.getErrMsg());
+ }
+ clearDirsForFinish.add(oldPartitionPath);
+ status = fs.renameDir(
+ writePath,
+ targetPath,
+ () -> directoryCleanUpTasksForAbort.add(new DirectoryCleanUpTask(targetPath, true)));
+ if (!status.ok()) {
+ throw new RuntimeException(
+ "Error to rename dir from " + writePath + " to " + targetPath + ":" + status.getErrMsg());
+ }
+ }
+ updateStatisticsTasks.add(
+ new UpdateStatisticsTask(
+ table.getDbName(),
+ table.getTableName(),
+ Optional.of(pu.getName()),
+ ps,
+ false
+ ));
}
@@ -481,7 +547,7 @@ public class HMSCommitter {
createdPartitionValues.add(partition.getPartition().getPartitionValues());
}
} catch (Throwable t) {
- LOG.error("Failed to add partition", t);
+ LOG.warn("Failed to add partition", t);
throw t;
}
}
@@ -583,7 +649,27 @@ public class HMSCommitter {
}
private void runRenameDirTasksForAbort() {
- // TODO abort
+ Status status;
+ for (RenameDirectoryTask task : renameDirectoryTasksForAbort) {
+ status = fs.exists(task.getRenameFrom());
+ if (status.ok()) {
+ status = fs.renameDir(task.getRenameFrom(), task.getRenameTo(), () -> {});
+ if (!status.ok()) {
+ LOG.warn("Failed to abort rename dir from {} to {}:{}",
+ task.getRenameFrom(), task.getRenameTo(), status.getErrMsg());
+ }
+ }
+ }
+ }
+
+ private void runClearPathsForFinish() {
+ Status status;
+ for (String path : clearDirsForFinish) {
+ status = fs.delete(path);
+ if (!status.ok()) {
+ LOG.warn("Failed to recursively delete path {}:{}", path, status.getErrCode());
+ }
+ }
}
@@ -591,10 +677,10 @@ public class HMSCommitter {
DeleteRecursivelyResult deleteResult = recursiveDeleteFiles(directory, deleteEmptyDir);
if (!deleteResult.getNotDeletedEligibleItems().isEmpty()) {
- LOG.error("Failed to delete directory {}. Some eligible items can't be deleted: {}.",
+ LOG.warn("Failed to delete directory {}. Some eligible items can't be deleted: {}.",
directory.toString(), deleteResult.getNotDeletedEligibleItems());
} else if (deleteEmptyDir && !deleteResult.dirNotExists()) {
- LOG.error("Failed to delete directory {} due to dir isn't empty", directory.toString());
+ LOG.warn("Failed to delete directory {} due to dir isn't empty", directory.toString());
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystem.java
index 798f93a61c0..0470d8b3714 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystem.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystem.java
@@ -49,6 +49,12 @@ public interface FileSystem {
Status rename(String origFilePath, String destFilePath);
+ default Status renameDir(String origFilePath,
+ String destFilePath,
+ Runnable runWhenPathNotExist) {
+ throw new UnsupportedOperationException("Unsupported operation rename dir on current file system.");
+ }
+
default void asyncRename(Executor executor,
List<CompletableFuture<?>> renameFileFutures,
AtomicBoolean cancelled,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java
index e8c645f3c9b..26008adf38c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java
@@ -412,6 +412,28 @@ public class DFSFileSystem extends RemoteFileSystem {
}
}
+ public Status renameDir(String origFilePath,
+ String destFilePath,
+ Runnable runWhenPathNotExist) {
+ Status status = exists(destFilePath);
+ if (status.ok()) {
+ throw new RuntimeException("Destination directory already exists: " + destFilePath);
+ }
+
+ String targetParent = new Path(destFilePath).getParent().toString();
+ status = exists(targetParent);
+ if (Status.ErrCode.NOT_FOUND.equals(status.getErrCode())) {
+ status = makeDir(targetParent);
+ }
+ if (!status.ok()) {
+ throw new RuntimeException(status.getErrMsg());
+ }
+
+ runWhenPathNotExist.run();
+
+ return rename(origFilePath, destFilePath);
+ }
+
@Override
public void asyncRenameDir(Executor executor,
List<CompletableFuture<?>> renameFileFutures,
@@ -423,23 +445,7 @@ public class DFSFileSystem extends RemoteFileSystem {
if (cancelled.get()) {
return;
}
-
- Status status = exists(destFilePath);
- if (status.ok()) {
- throw new RuntimeException("Destination directory already exists: " + destFilePath);
- }
-
- String targetParent = new Path(destFilePath).getParent().toString();
- status = exists(targetParent);
- if (Status.ErrCode.NOT_FOUND.equals(status.getErrCode())) {
- makeDir(targetParent);
- } else if (!status.ok()) {
- throw new RuntimeException(status.getErrMsg());
- }
-
- runWhenPathNotExist.run();
-
- status = rename(origFilePath, destFilePath);
+ Status status = renameDir(origFilePath, destFilePath, runWhenPathNotExist);
if (!status.ok()) {
throw new RuntimeException(status.getErrMsg());
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HmsCommitTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HmsCommitTest.java
index 2316e65bf60..1cfb2270f92 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HmsCommitTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HmsCommitTest.java
@@ -134,7 +134,7 @@ public class HmsCommitTest {
pus.add(createRandomAppend(""));
hmsOps.commit(dbName, tbWithoutPartition, pus);
Table table = hmsClient.getTable(dbName, tbWithoutPartition);
- Assert.assertEquals(3, Long.parseLong(table.getParameters().get("numRows")));
+ assertNumRows(3, table);
List<THivePartitionUpdate> pus2 = new ArrayList<>();
pus2.add(createRandomAppend(""));
@@ -142,12 +142,19 @@ public class HmsCommitTest {
pus2.add(createRandomAppend(""));
hmsOps.commit(dbName, tbWithoutPartition, pus2);
table = hmsClient.getTable(dbName, tbWithoutPartition);
- Assert.assertEquals(6, Long.parseLong(table.getParameters().get("numRows")));
+ assertNumRows(6, table);
}
@Test
public void testOverwritePartitionForUnPartitionedTable() {
- // TODO
+ testAppendPartitionForUnPartitionedTable();
+ List<THivePartitionUpdate> pus = new ArrayList<>();
+ pus.add(createRandomOverwrite(""));
+ pus.add(createRandomOverwrite(""));
+ pus.add(createRandomOverwrite(""));
+ hmsOps.commit(dbName, tbWithoutPartition, pus);
+ Table table = hmsClient.getTable(dbName, tbWithoutPartition);
+ assertNumRows(3, table);
}
@Test
@@ -162,11 +169,11 @@ public class HmsCommitTest {
hmsOps.commit(dbName, tbWithPartition, pus);
Partition pa = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("a"));
- Assert.assertEquals(3, Long.parseLong(pa.getParameters().get("numRows")));
+ assertNumRows(3, pa);
Partition pb = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("b"));
- Assert.assertEquals(2, Long.parseLong(pb.getParameters().get("numRows")));
+ assertNumRows(2, pb);
Partition pc = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("c"));
- Assert.assertEquals(1, Long.parseLong(pc.getParameters().get("numRows")));
+ assertNumRows(1, pc);
}
@Test
@@ -183,11 +190,28 @@ public class HmsCommitTest {
hmsOps.commit(dbName, tbWithPartition, pus);
Partition pa = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("a"));
- Assert.assertEquals(6, Long.parseLong(pa.getParameters().get("numRows")));
+ assertNumRows(6, pa);
+ Partition pb = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("b"));
+ assertNumRows(4, pb);
+ Partition pc = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("c"));
+ assertNumRows(2, pc);
+ }
+
+ @Test
+ public void testOverwritePartitionForPartitionedTable() {
+ testAppendPartitionForPartitionedTable();
+ List<THivePartitionUpdate> pus = new ArrayList<>();
+ pus.add(createRandomOverwrite("a"));
+ pus.add(createRandomOverwrite("b"));
+ pus.add(createRandomOverwrite("c"));
+ hmsOps.commit(dbName, tbWithPartition, pus);
+
+ Partition pa = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("a"));
+ assertNumRows(1, pa);
Partition pb = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("b"));
- Assert.assertEquals(4, Long.parseLong(pb.getParameters().get("numRows")));
+ assertNumRows(1, pb);
Partition pc = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("c"));
- Assert.assertEquals(2, Long.parseLong(pc.getParameters().get("numRows")));
+ assertNumRows(1, pc);
}
@Test
@@ -201,7 +225,7 @@ public class HmsCommitTest {
hmsOps.commit(dbName, tbWithPartition, pus);
for (int i = 0; i < nums; i++) {
Partition p = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("" + i));
- Assert.assertEquals(1, Long.parseLong(p.getParameters().get("numRows")));
+ assertNumRows(1, p);
}
try {
@@ -211,6 +235,14 @@ public class HmsCommitTest {
}
}
+ public void assertNumRows(long expected, Partition p) {
+ Assert.assertEquals(expected, Long.parseLong(p.getParameters().get("numRows")));
+ }
+
+ public void assertNumRows(long expected, Table t) {
+ Assert.assertEquals(expected, Long.parseLong(t.getParameters().get("numRows")));
+ }
+
public THivePartitionUpdate genOnePartitionUpdate(String partitionValue, TUpdateMode mode) {
String uuid = UUID.randomUUID().toString();
@@ -242,4 +274,8 @@ public class HmsCommitTest {
public THivePartitionUpdate createRandomAppend(String partition) {
return genOnePartitionUpdate("c3=" + partition, TUpdateMode.APPEND);
}
+
+ public THivePartitionUpdate createRandomOverwrite(String partition) {
+ return genOnePartitionUpdate("c3=" + partition, TUpdateMode.OVERWRITE);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 05/15: [test](fuzzy) test fuzzy in BE (#31607)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 4ebbabf15e8fb352e7b8066e2fbb0ce648dbdd42
Author: zhangstar333 <87...@users.noreply.github.com>
AuthorDate: Fri Mar 22 19:43:26 2024 +0800
[test](fuzzy) test fuzzy in BE (#31607)
test fuzzy in BE
---
be/src/common/config.cpp | 47 +++++++++++++++--------------------------------
be/src/common/config.h | 24 ------------------------
2 files changed, 15 insertions(+), 56 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index ab7bd39e034..c73e3670b3f 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -32,6 +32,7 @@
#include <map>
#include <memory>
#include <mutex>
+#include <random>
#include <string>
#include <utility>
#include <vector>
@@ -309,7 +310,6 @@ DEFINE_mInt32(trash_file_expire_time_sec, "259200");
// modify them upon necessity
DEFINE_Int32(min_file_descriptor_number, "60000");
DEFINE_mBool(disable_segment_cache, "false");
-DEFINE_Int64(index_stream_cache_capacity, "10737418240");
DEFINE_String(row_cache_mem_limit, "20%");
// Cache for storage page size
@@ -466,8 +466,6 @@ DEFINE_String(ssl_private_key_path, "");
DEFINE_Bool(enable_all_http_auth, "false");
// Number of webserver workers
DEFINE_Int32(webserver_num_workers, "48");
-// Period to update rate counters and sampling counters in ms.
-DEFINE_mInt32(periodic_counter_update_period_ms, "500");
DEFINE_Bool(enable_single_replica_load, "true");
// Number of download workers for single replica load
@@ -535,10 +533,6 @@ DEFINE_Int32(num_threads_per_disk, "0");
DEFINE_Int32(read_size, "8388608"); // 8 * 1024 * 1024, Read Size (in bytes)
DEFINE_Int32(min_buffer_size, "1024"); // 1024, The minimum read buffer size (in bytes)
-// For each io buffer size, the maximum number of buffers the IoMgr will hold onto
-// With 1024B through 8MB buffers, this is up to ~2GB of buffers.
-DEFINE_Int32(max_free_io_buffers, "128");
-
// for pprof
DEFINE_String(pprof_profile_dir, "${DORIS_HOME}/log");
// for jeprofile in jemalloc
@@ -559,12 +553,6 @@ DEFINE_Int32(num_cores, "0");
// Otherwise, we will ignore the broken disk,
DEFINE_Bool(ignore_broken_disk, "false");
-// linux transparent huge page
-DEFINE_Bool(madvise_huge_pages, "false");
-
-// whether use mmap to allocate memory
-DEFINE_Bool(mmap_buffers, "false");
-
// Sleep time in milliseconds between memory maintenance iterations
DEFINE_mInt32(memory_maintenance_sleep_time_ms, "100");
@@ -581,9 +569,6 @@ DEFINE_mInt32(memtable_hard_limit_active_percent, "50");
// percent of (active memtables size / all memtables size) when reach soft limit
DEFINE_mInt32(memtable_soft_limit_active_percent, "50");
-// Alignment
-DEFINE_Int32(memory_max_alignment, "16");
-
// memtable insert memory tracker will multiply input block size with this ratio
DEFINE_mDouble(memtable_insert_memory_ratio, "1.4");
// max write buffer size before flush, default 200MB
@@ -809,9 +794,6 @@ DEFINE_mInt32(jdbc_connection_pool_cache_clear_time_sec, "28800");
DEFINE_Int64(delete_bitmap_agg_cache_capacity, "104857600");
DEFINE_mInt32(delete_bitmap_agg_cache_stale_sweep_time_sec, "1800");
-// s3 config
-DEFINE_mInt32(max_remote_storage_count, "10");
-
// reference https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md#broker-version-compatibility
// If the dependent kafka broker version older than 0.10.0.0,
// the value of kafka_api_version_request should be false, and the
@@ -897,14 +879,6 @@ DEFINE_mInt32(remove_unused_remote_files_interval_sec, "21600"); // 6h
DEFINE_mInt32(confirm_unused_remote_files_interval_sec, "60");
DEFINE_Int32(cold_data_compaction_thread_num, "2");
DEFINE_mInt32(cold_data_compaction_interval_sec, "1800");
-DEFINE_Int32(concurrency_per_dir, "2");
-// file_cache_type is used to set the type of file cache for remote files.
-// "": no cache, "sub_file_cache": split sub files from remote file.
-// "whole_file_cache": the whole file.
-DEFINE_mString(file_cache_type, "file_block_cache");
-DEFINE_Validator(file_cache_type, [](std::string_view config) -> bool {
- return config.empty() || config == "file_block_cache";
-});
DEFINE_String(tmp_file_dir, "tmp");
@@ -1638,15 +1612,24 @@ void update_config(const std::string& field, const std::string& value) {
Status set_fuzzy_configs() {
std::unordered_map<std::string, std::string> fuzzy_field_and_value;
+ std::shared_ptr<std::mt19937_64> generator(new std::mt19937_64());
+ generator->seed(std::random_device()());
+ std::uniform_int_distribution<int64_t> distribution(0, 100);
// if have set enable_fuzzy_mode=true in be.conf, will fuzzy those field and values
- fuzzy_field_and_value["disable_storage_page_cache"] = ((rand() % 2) == 0) ? "true" : "false";
- fuzzy_field_and_value["enable_system_metrics"] = ((rand() % 2) == 0) ? "true" : "false";
+ fuzzy_field_and_value["disable_storage_page_cache"] =
+ ((distribution(*generator) % 2) == 0) ? "true" : "false";
+ fuzzy_field_and_value["enable_system_metrics"] =
+ ((distribution(*generator) % 2) == 0) ? "true" : "false";
+ fuzzy_field_and_value["enable_set_in_bitmap_value"] =
+ ((distribution(*generator) % 2) == 0) ? "true" : "false";
+ fuzzy_field_and_value["enable_shrink_memory"] =
+ ((distribution(*generator) % 2) == 0) ? "true" : "false";
fmt::memory_buffer buf;
- for (auto it = fuzzy_field_and_value.begin(); it != fuzzy_field_and_value.end(); it++) {
- const auto& field = it->first;
- const auto& value = it->second;
+ for (auto& it : fuzzy_field_and_value) {
+ const auto& field = it.first;
+ const auto& value = it.second;
RETURN_IF_ERROR(set_config(field, value, false, true));
fmt::format_to(buf, "{}={}, ", field, value);
}
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 4aaed552d89..ae4ff9c410e 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -353,7 +353,6 @@ DECLARE_mInt32(trash_file_expire_time_sec);
// modify them upon necessity
DECLARE_Int32(min_file_descriptor_number);
DECLARE_mBool(disable_segment_cache);
-DECLARE_Int64(index_stream_cache_capacity);
DECLARE_String(row_cache_mem_limit);
// Cache for storage page size
@@ -509,8 +508,6 @@ DECLARE_String(ssl_private_key_path);
DECLARE_Bool(enable_all_http_auth);
// Number of webserver workers
DECLARE_Int32(webserver_num_workers);
-// Period to update rate counters and sampling counters in ms.
-DECLARE_mInt32(periodic_counter_update_period_ms);
DECLARE_Bool(enable_single_replica_load);
// Number of download workers for single replica load
@@ -585,10 +582,6 @@ DECLARE_Int32(num_threads_per_disk);
DECLARE_Int32(read_size); // 8 * 1024 * 1024, Read Size (in bytes)
DECLARE_Int32(min_buffer_size); // 1024, The minimum read buffer size (in bytes)
-// For each io buffer size, the maximum number of buffers the IoMgr will hold onto
-// With 1024B through 8MB buffers, this is up to ~2GB of buffers.
-DECLARE_Int32(max_free_io_buffers);
-
// for pprof
DECLARE_String(pprof_profile_dir);
// for jeprofile in jemalloc
@@ -610,12 +603,6 @@ DECLARE_Int32(num_cores);
// Otherwise, we will ignore the broken disk,
DECLARE_Bool(ignore_broken_disk);
-// linux transparent huge page
-DECLARE_Bool(madvise_huge_pages);
-
-// whether use mmap to allocate memory
-DECLARE_Bool(mmap_buffers);
-
// Sleep time in milliseconds between memory maintenance iterations
DECLARE_mInt32(memory_maintenance_sleep_time_ms);
@@ -632,9 +619,6 @@ DECLARE_mInt32(memtable_hard_limit_active_percent);
// percent of (active memtables size / all memtables size) when reach soft limit
DECLARE_mInt32(memtable_soft_limit_active_percent);
-// Alignment
-DECLARE_Int32(memory_max_alignment);
-
// memtable insert memory tracker will multiply input block size with this ratio
DECLARE_mDouble(memtable_insert_memory_ratio);
// max write buffer size before flush, default 200MB
@@ -866,9 +850,6 @@ DECLARE_mInt32(delete_bitmap_agg_cache_stale_sweep_time_sec);
// A common object cache depends on an Sharded LRU Cache.
DECLARE_mInt32(common_obj_lru_cache_stale_sweep_time_sec);
-// s3 config
-DECLARE_mInt32(max_remote_storage_count);
-
// reference https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md#broker-version-compatibility
// If the dependent kafka broker version older than 0.10.0.0,
// the value of kafka_api_version_request should be false, and the
@@ -951,11 +932,6 @@ DECLARE_mInt32(remove_unused_remote_files_interval_sec); // 6h
DECLARE_mInt32(confirm_unused_remote_files_interval_sec);
DECLARE_Int32(cold_data_compaction_thread_num);
DECLARE_mInt32(cold_data_compaction_interval_sec);
-DECLARE_Int32(concurrency_per_dir);
-// file_cache_type is used to set the type of file cache for remote files.
-// "": no cache, "sub_file_cache": split sub files from remote file.
-// "whole_file_cache": the whole file.
-DECLARE_mString(file_cache_type);
DECLARE_Int32(s3_transfer_executor_pool_size);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 06/15: [fix](Nereids) fix bind having aggregate failed again (#32687)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 3d14d9e379d15040db9d6bab1e028948f590fb92
Author: 924060929 <92...@qq.com>
AuthorDate: Fri Mar 22 20:39:40 2024 +0800
[fix](Nereids) fix bind having aggregate failed again (#32687)
follow up #32490
add more tests and fix some cases because some sqls are valid to mysql, but failed in doris
---
.../nereids/rules/analysis/BindExpression.java | 37 ++++------------------
.../data/nereids_syntax_p0/bind_priority.out | 9 ++++++
.../suites/nereids_syntax_p0/bind_priority.groovy | 21 ++++++++++++
3 files changed, 37 insertions(+), 30 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java
index 04ba599aa60..c2c7f5815d9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java
@@ -19,7 +19,6 @@ package org.apache.doris.nereids.rules.analysis;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.FunctionRegistry;
-import org.apache.doris.common.Pair;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.StatementContext;
@@ -386,41 +385,19 @@ public class BindExpression implements AnalysisRuleFactory {
}
Scope groupBySlotsScope = toScope(cascadesContext, groupBySlots.build());
- Supplier<Pair<Scope, Scope>> separateAggOutputScopes = Suppliers.memoize(() -> {
- ImmutableList.Builder<Slot> groupByOutputs = ImmutableList.builderWithExpectedSize(
- aggregate.getOutputExpressions().size());
- ImmutableList.Builder<Slot> aggFunOutputs = ImmutableList.builderWithExpectedSize(
- aggregate.getOutputExpressions().size());
- for (NamedExpression outputExpression : aggregate.getOutputExpressions()) {
- if (outputExpression.anyMatch(AggregateFunction.class::isInstance)) {
- aggFunOutputs.add(outputExpression.toSlot());
- } else {
- groupByOutputs.add(outputExpression.toSlot());
- }
- }
- Scope nonAggFunSlotsScope = toScope(cascadesContext, groupByOutputs.build());
- Scope aggFuncSlotsScope = toScope(cascadesContext, aggFunOutputs.build());
- return Pair.of(nonAggFunSlotsScope, aggFuncSlotsScope);
- });
-
return (analyzer, unboundSlot) -> {
List<Slot> boundInGroupBy = analyzer.bindSlotByScope(unboundSlot, groupBySlotsScope);
- if (boundInGroupBy.size() == 1) {
- return boundInGroupBy;
- }
-
- Pair<Scope, Scope> separateAggOutputScope = separateAggOutputScopes.get();
- List<Slot> boundInNonAggFuncs = analyzer.bindSlotByScope(unboundSlot, separateAggOutputScope.first);
- if (boundInNonAggFuncs.size() == 1) {
- return boundInNonAggFuncs;
+ if (!boundInGroupBy.isEmpty()) {
+ return ImmutableList.of(boundInGroupBy.get(0));
}
- List<Slot> boundInAggFuncs = analyzer.bindSlotByScope(unboundSlot, separateAggOutputScope.second);
- if (boundInAggFuncs.size() == 1) {
- return boundInAggFuncs;
+ List<Slot> boundInAggOutput = analyzer.bindSlotByScope(unboundSlot, aggOutputScope);
+ if (!boundInAggOutput.isEmpty()) {
+ return ImmutableList.of(boundInAggOutput.get(0));
}
- return bindByAggChild.get().bindSlot(analyzer, unboundSlot);
+ List<? extends Expression> expressions = bindByAggChild.get().bindSlot(analyzer, unboundSlot);
+ return expressions.isEmpty() ? expressions : ImmutableList.of(expressions.get(0));
};
});
diff --git a/regression-test/data/nereids_syntax_p0/bind_priority.out b/regression-test/data/nereids_syntax_p0/bind_priority.out
index eb4002d960a..53432880c24 100644
--- a/regression-test/data/nereids_syntax_p0/bind_priority.out
+++ b/regression-test/data/nereids_syntax_p0/bind_priority.out
@@ -76,3 +76,12 @@ all 2
-- !having_bind_group_by --
7 3
+-- !having_bind_group_by --
+4 5 3
+
+-- !having_bind_group_by --
+1 2
+
+-- !having_bind_group_by --
+2 1
+
diff --git a/regression-test/suites/nereids_syntax_p0/bind_priority.groovy b/regression-test/suites/nereids_syntax_p0/bind_priority.groovy
index 4e1740061b6..84bab14eba0 100644
--- a/regression-test/suites/nereids_syntax_p0/bind_priority.groovy
+++ b/regression-test/suites/nereids_syntax_p0/bind_priority.groovy
@@ -287,5 +287,26 @@ suite("bind_priority") {
GROUP by pk + 6
HAVING pk = 3
"""
+
+ order_qt_having_bind_group_by """
+ select pk + 1 as pk, pk + 2 as pk, count(*)
+ from test_bind_having_slots3
+ group by pk + 1, pk + 2
+ having pk = 4;
+ """
+
+ order_qt_having_bind_group_by """
+ select count(*) pk, pk + 1 as pk
+ from test_bind_having_slots3
+ group by pk + 1, pk + 2
+ having pk = 1;
+ """
+
+ order_qt_having_bind_group_by """
+ select pk + 1 as pk, count(*) pk
+ from test_bind_having_slots3
+ group by pk + 1, pk + 2
+ having pk = 2;
+ """
}()
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 01/15: Add log for case to help debug. (#32675)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 735f5213c10f225ceee28b2ad264a2a3b3eafa69
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Fri Mar 22 17:14:15 2024 +0800
Add log for case to help debug. (#32675)
---
.../suites/statistics/analyze_stats.groovy | 23 ++++++++++++++++++++++
1 file changed, 23 insertions(+)
diff --git a/regression-test/suites/statistics/analyze_stats.groovy b/regression-test/suites/statistics/analyze_stats.groovy
index 8d90b67632a..699e595df59 100644
--- a/regression-test/suites/statistics/analyze_stats.groovy
+++ b/regression-test/suites/statistics/analyze_stats.groovy
@@ -1269,6 +1269,7 @@ PARTITION `p599` VALUES IN (599)
assertEquals(14, result_after_truncate.size())
result = sql """show column stats ${tbl}(analyzetestlimitedk0);"""
+ logger.info("show column analyzetestlimitedk0 stats: " + result)
assertEquals(1, result.size())
assertEquals("analyzetestlimitedk0", result[0][0])
assertEquals("0.0", result[0][2])
@@ -1280,6 +1281,7 @@ PARTITION `p599` VALUES IN (599)
assertEquals("N/A", result[0][8])
result = sql """show column stats ${tbl}(analyzetestlimitedk1);"""
+ logger.info("show column analyzetestlimitedk1 stats: " + result)
assertEquals(1, result.size())
assertEquals("analyzetestlimitedk1", result[0][0])
assertEquals("0.0", result[0][2])
@@ -1291,6 +1293,7 @@ PARTITION `p599` VALUES IN (599)
assertEquals("N/A", result[0][8])
result = sql """show column stats ${tbl}(analyzetestlimitedk2);"""
+ logger.info("show column analyzetestlimitedk2 stats: " + result)
assertEquals(1, result.size())
assertEquals("analyzetestlimitedk2", result[0][0])
assertEquals("0.0", result[0][2])
@@ -1302,6 +1305,7 @@ PARTITION `p599` VALUES IN (599)
assertEquals("N/A", result[0][8])
result = sql """show column stats ${tbl}(analyzetestlimitedk3);"""
+ logger.info("show column analyzetestlimitedk3 stats: " + result)
assertEquals(1, result.size())
assertEquals("analyzetestlimitedk3", result[0][0])
assertEquals("0.0", result[0][2])
@@ -1313,6 +1317,7 @@ PARTITION `p599` VALUES IN (599)
assertEquals("N/A", result[0][8])
result = sql """show column stats ${tbl}(analyzetestlimitedk4);"""
+ logger.info("show column analyzetestlimitedk4 stats: " + result)
assertEquals(1, result.size())
assertEquals("analyzetestlimitedk4", result[0][0])
assertEquals("0.0", result[0][2])
@@ -1324,6 +1329,7 @@ PARTITION `p599` VALUES IN (599)
assertEquals("N/A", result[0][8])
result = sql """show column stats ${tbl}(analyzetestlimitedk5);"""
+ logger.info("show column analyzetestlimitedk5 stats: " + result)
assertEquals(1, result.size())
assertEquals("analyzetestlimitedk5", result[0][0])
assertEquals("0.0", result[0][2])
@@ -1335,6 +1341,7 @@ PARTITION `p599` VALUES IN (599)
assertEquals("N/A", result[0][8])
result = sql """show column stats ${tbl}(analyzetestlimitedk6);"""
+ logger.info("show column analyzetestlimitedk6 stats: " + result)
assertEquals(1, result.size())
assertEquals("analyzetestlimitedk6", result[0][0])
assertEquals("0.0", result[0][2])
@@ -1346,6 +1353,7 @@ PARTITION `p599` VALUES IN (599)
assertEquals("N/A", result[0][8])
result = sql """show column stats ${tbl}(analyzetestlimitedk7);"""
+ logger.info("show column analyzetestlimitedk7 stats: " + result)
assertEquals(1, result.size())
assertEquals("analyzetestlimitedk7", result[0][0])
assertEquals("0.0", result[0][2])
@@ -1357,6 +1365,7 @@ PARTITION `p599` VALUES IN (599)
assertEquals("N/A", result[0][8])
result = sql """show column stats ${tbl}(analyzetestlimitedk8);"""
+ logger.info("show column analyzetestlimitedk8 stats: " + result)
assertEquals(1, result.size())
assertEquals("analyzetestlimitedk8", result[0][0])
assertEquals("0.0", result[0][2])
@@ -1368,6 +1377,7 @@ PARTITION `p599` VALUES IN (599)
assertEquals("N/A", result[0][8])
result = sql """show column stats ${tbl}(analyzetestlimitedk9);"""
+ logger.info("show column analyzetestlimitedk9 stats: " + result)
assertEquals(1, result.size())
assertEquals("analyzetestlimitedk9", result[0][0])
assertEquals("0.0", result[0][2])
@@ -1379,6 +1389,7 @@ PARTITION `p599` VALUES IN (599)
assertEquals("N/A", result[0][8])
result = sql """show column stats ${tbl}(analyzetestlimitedk10);"""
+ logger.info("show column analyzetestlimitedk10 stats: " + result)
assertEquals(1, result.size())
assertEquals("analyzetestlimitedk10", result[0][0])
assertEquals("0.0", result[0][2])
@@ -1390,6 +1401,7 @@ PARTITION `p599` VALUES IN (599)
assertEquals("N/A", result[0][8])
result = sql """show column stats ${tbl}(analyzetestlimitedk11);"""
+ logger.info("show column analyzetestlimitedk11 stats: " + result)
assertEquals(1, result.size())
assertEquals("analyzetestlimitedk11", result[0][0])
assertEquals("0.0", result[0][2])
@@ -1401,6 +1413,7 @@ PARTITION `p599` VALUES IN (599)
assertEquals("N/A", result[0][8])
result = sql """show column stats ${tbl}(analyzetestlimitedk12);"""
+ logger.info("show column analyzetestlimitedk12 stats: " + result)
assertEquals(1, result.size())
assertEquals("analyzetestlimitedk12", result[0][0])
assertEquals("0.0", result[0][2])
@@ -1412,6 +1425,7 @@ PARTITION `p599` VALUES IN (599)
assertEquals("N/A", result[0][8])
result = sql """show column stats ${tbl}(analyzetestlimitedk13);"""
+ logger.info("show column analyzetestlimitedk13 stats: " + result)
assertEquals(1, result.size())
assertEquals("analyzetestlimitedk13", result[0][0])
assertEquals("0.0", result[0][2])
@@ -2555,6 +2569,7 @@ PARTITION `p599` VALUES IN (599)
sql """insert into region values(3,'name3', 'comment3') """
sql """ANALYZE TABLE region WITH SYNC"""
result = sql """show column stats region (`r%name`);"""
+ logger.info("show column r%name stats: " + result)
assertEquals(1, result.size())
assertEquals("r%name", result[0][0])
assertEquals("3.0", result[0][2])
@@ -2616,6 +2631,7 @@ PARTITION `p599` VALUES IN (599)
Thread.sleep(1000 * 60)
sql """analyze table agg_table_test with sample rows 100 with sync"""
def agg_result = sql """show column stats agg_table_test (name)"""
+ logger.info("show column agg_table_test(name) stats: " + agg_result)
assertEquals(agg_result[0][7], "N/A")
assertEquals(agg_result[0][8], "N/A")
@@ -2674,6 +2690,7 @@ PARTITION `p599` VALUES IN (599)
assertEquals("false", alter_result[0][7])
sql """alter table alter_test modify column id set stats ('row_count'='100', 'ndv'='0', 'num_nulls'='0.0', 'data_size'='2.69975443E8', 'min_value'='1', 'max_value'='2');"""
alter_result = sql """show column stats alter_test(id)"""
+ logger.info("show column alter_test(id) stats: " + alter_result)
assertEquals(1, alter_result.size())
alter_result = sql """show column cached stats alter_test(id)"""
assertEquals(0, alter_result.size())
@@ -2681,11 +2698,13 @@ PARTITION `p599` VALUES IN (599)
assertEquals(0, alter_result.size())
sql """alter table alter_test modify column id set stats ('row_count'='100', 'ndv'='0', 'num_nulls'='100', 'data_size'='2.69975443E8', 'min_value'='1', 'max_value'='2');"""
alter_result = sql """show column stats alter_test(id)"""
+ logger.info("show column alter_test(id) stats: " + alter_result)
assertEquals(1, alter_result.size())
alter_result = sql """show column cached stats alter_test(id)"""
assertEquals(1, alter_result.size())
sql """alter table alter_test modify column id set stats ('row_count'='100', 'ndv'='1', 'num_nulls'='0', 'data_size'='2.69975443E8', 'min_value'='1', 'max_value'='2');"""
alter_result = sql """show column stats alter_test(id)"""
+ logger.info("show column alter_test(id) stats: " + alter_result)
assertEquals(1, alter_result.size())
alter_result = sql """show column cached stats alter_test(id)"""
assertEquals(1, alter_result.size())
@@ -2709,6 +2728,7 @@ PARTITION `p599` VALUES IN (599)
// Test sample empty table
def result_sample = sql """analyze table trigger_test with sample percent 10 with sync"""
result_sample = sql """show column stats trigger_test"""
+ logger.info("show column trigger_test stats: " + result_sample)
assertEquals(2, result_sample.size())
assertEquals("0.0", result_sample[0][2])
assertEquals("SAMPLE", result_sample[0][9])
@@ -2718,6 +2738,7 @@ PARTITION `p599` VALUES IN (599)
sql """drop stats trigger_test"""
sql """analyze table trigger_test with sample rows 1000 with sync"""
result_sample = sql """show column stats trigger_test"""
+ logger.info("show column trigger_test stats: " + result_sample)
assertEquals(2, result_sample.size())
assertEquals("0.0", result_sample[0][2])
assertEquals("SAMPLE", result_sample[0][9])
@@ -2734,6 +2755,7 @@ PARTITION `p599` VALUES IN (599)
// Test analyze default full.
sql """analyze table trigger_test with sync"""
def result = sql """show column stats trigger_test"""
+ logger.info("show column trigger_test stats: " + result)
assertEquals(2, result.size())
assertEquals("4.0", result[0][2])
assertEquals("FULL", result[0][9])
@@ -2744,6 +2766,7 @@ PARTITION `p599` VALUES IN (599)
sql """insert into trigger_test values(5,'name5') """
sql """analyze table trigger_test with sync"""
result = sql """show column stats trigger_test"""
+ logger.info("show column trigger_test stats: " + result)
assertEquals(2, result.size())
assertEquals("5.0", result[0][2])
assertEquals("5.0", result[1][2])
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 07/15: [fix](Nereids) do not push down topn-filter through right/full outer join if the first orderkey is nulls first (#32633)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit f62cdecc08bdc6cd3ec691f885034adcaa0706fd
Author: minghong <en...@gmail.com>
AuthorDate: Fri Mar 22 20:53:30 2024 +0800
[fix](Nereids) do not push down topn-filter through right/full outer join if the first orderkey is nulls first (#32633)
* do not push down topn-filter through right/full outer join if the first order key is nulls first
---
.../apache/doris/nereids/processor/post/TopNScanOpt.java | 16 +++++++++++++---
.../suites/nereids_tpch_p0/tpch/topn-filter.groovy | 15 +++++++++++++++
2 files changed, 28 insertions(+), 3 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopNScanOpt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopNScanOpt.java
index a9425cb715b..63754afe4df 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopNScanOpt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopNScanOpt.java
@@ -22,6 +22,7 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.SortPhase;
+import org.apache.doris.nereids.trees.plans.algebra.Join;
import org.apache.doris.nereids.trees.plans.algebra.OlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeTopN;
@@ -89,7 +90,9 @@ public class TopNScanOpt extends PlanPostProcessor {
if (! (firstKey instanceof SlotReference)) {
return Optional.empty();
}
- OlapScan olapScan = findScanNodeBySlotReference(topN, (SlotReference) firstKey);
+
+ boolean nullsFirst = topN.getOrderKeys().get(0).isNullFirst();
+ OlapScan olapScan = findScanNodeBySlotReference(topN, (SlotReference) firstKey, nullsFirst);
if (olapScan != null
&& olapScan.getTable().isDupKeysOrMergeOnWrite()
&& olapScan instanceof PhysicalCatalogRelation) {
@@ -99,7 +102,14 @@ public class TopNScanOpt extends PlanPostProcessor {
return Optional.empty();
}
- private OlapScan findScanNodeBySlotReference(Plan root, SlotReference slot) {
+ private OlapScan findScanNodeBySlotReference(Plan root, SlotReference slot, boolean nullsFirst) {
+ // topn-filter cannot be pushed through right/full outer join if the first orderKey is nulls first
+ if (nullsFirst && root instanceof Join) {
+ Join join = (Join) root;
+ if (join.getJoinType().isRightOuterJoin() || join.getJoinType().isFullOuterJoin()) {
+ return null;
+ }
+ }
OlapScan target = null;
if (root instanceof OlapScan && root.getOutputSet().contains(slot)) {
return (OlapScan) root;
@@ -109,7 +119,7 @@ public class TopNScanOpt extends PlanPostProcessor {
// TODO for union, topn-filter can be pushed down to all of its children.
Plan child = root.child(0);
if (!(child instanceof PhysicalWindow) && child.getOutputSet().contains(slot)) {
- target = findScanNodeBySlotReference(child, slot);
+ target = findScanNodeBySlotReference(child, slot, nullsFirst);
if (target != null) {
return target;
}
diff --git a/regression-test/suites/nereids_tpch_p0/tpch/topn-filter.groovy b/regression-test/suites/nereids_tpch_p0/tpch/topn-filter.groovy
index 14733d1303f..3a29274a2fe 100644
--- a/regression-test/suites/nereids_tpch_p0/tpch/topn-filter.groovy
+++ b/regression-test/suites/nereids_tpch_p0/tpch/topn-filter.groovy
@@ -116,6 +116,21 @@ suite("topn-filter") {
}
qt_groupingsets "select n_regionkey, sum(n_nationkey) from nation group by grouping sets((n_regionkey)) order by n_regionkey limit 2;"
+
+ explain {
+ sql "select * from nation right outer join region on r_regionkey = n_regionkey order by n_regionkey nulls first limit 1; "
+ notContains "TOPN OPT:"
+ }
+
+ explain {
+ sql "select * from nation right outer join region on r_regionkey = n_regionkey order by n_regionkey nulls last limit 1; "
+ contains "TOPN OPT:"
+ }
+
+ explain {
+ sql "select * from nation right outer join region on r_regionkey = n_regionkey order by n_regionkey nulls last, n_name nulls first limit 1; "
+ contains "TOPN OPT:"
+ }
sql "set enable_pipeline_engine=false;"
sql "set enable_pipeline_x_engine=false;"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 15/15: [fix](move-memtable) fix load timeout caused by lost wakeup (#32720)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 5c3ca0fbc2bd94af4b58b77829ec369612941c06
Author: Xin Liao <li...@126.com>
AuthorDate: Sat Mar 23 22:33:59 2024 +0800
[fix](move-memtable) fix load timeout caused by lost wakeup (#32720)
---
be/src/vec/sink/load_stream_stub.cpp | 25 ++++++++++---------------
1 file changed, 10 insertions(+), 15 deletions(-)
diff --git a/be/src/vec/sink/load_stream_stub.cpp b/be/src/vec/sink/load_stream_stub.cpp
index da0276fdd06..6eb91e46853 100644
--- a/be/src/vec/sink/load_stream_stub.cpp
+++ b/be/src/vec/sink/load_stream_stub.cpp
@@ -321,21 +321,16 @@ Status LoadStreamStub::close_wait(RuntimeState* state, int64_t timeout_ms) {
}
DCHECK(timeout_ms > 0) << "timeout_ms should be greator than 0";
std::unique_lock<bthread::Mutex> lock(_close_mutex);
- if (!_is_closed.load()) {
- auto timeout_sec = timeout_ms / 1000;
- while (!state->get_query_ctx()->is_cancelled() && timeout_sec > 0) {
- //the query maybe cancel, so need check after wait 1s
- timeout_sec = timeout_sec - 1;
- int ret = _close_cv.wait_for(lock, 1000000);
- if (ret == 0) {
- break;
- }
- if (timeout_sec <= 0) {
- return Status::InternalError(
- "stream close_wait timeout, timeout_ms={}, load_id={}, dst_id={}, "
- "stream_id={}",
- timeout_ms, print_id(_load_id), _dst_id, _stream_id);
- }
+ auto timeout_sec = timeout_ms / 1000;
+ while (!_is_closed.load() && !state->get_query_ctx()->is_cancelled()) {
+ //the query maybe cancel, so need check after wait 1s
+ timeout_sec = timeout_sec - 1;
+ int ret = _close_cv.wait_for(lock, 1000000);
+ if (ret != 0 && timeout_sec <= 0) {
+ return Status::InternalError(
+ "stream close_wait timeout, error={}, timeout_ms={}, load_id={}, dst_id={}, "
+ "stream_id={}",
+ ret, timeout_ms, print_id(_load_id), _dst_id, _stream_id);
}
}
RETURN_IF_ERROR(_check_cancel());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 04/15: [bug](udaf) fix memory leak in the java udaf (#32630)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit a55b5ea9caa3d2e75ab7f2a8e9ea8cb1f3c2830c
Author: zhangstar333 <87...@users.noreply.github.com>
AuthorDate: Fri Mar 22 19:43:14 2024 +0800
[bug](udaf) fix memory leak in the java udaf (#32630)
fix memory leak in the java udaf
---
.../aggregate_function_java_udaf.h | 35 ++++++++++++++++------
1 file changed, 26 insertions(+), 9 deletions(-)
diff --git a/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h b/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h
index 7a367623eaf..59711d513dd 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h
@@ -59,18 +59,28 @@ public:
AggregateJavaUdafData() = default;
AggregateJavaUdafData(int64_t num_args) { argument_size = num_args; }
- ~AggregateJavaUdafData() {
+ ~AggregateJavaUdafData() = default;
+
+ Status close_and_delete_object() {
JNIEnv* env = nullptr;
- if (!JniUtil::GetJNIEnv(&env).ok()) {
+ Defer defer {[&]() {
+ if (env != nullptr) {
+ env->DeleteGlobalRef(executor_cl);
+ env->DeleteGlobalRef(executor_obj);
+ }
+ }};
+ Status st = JniUtil::GetJNIEnv(&env);
+ if (!st.ok()) {
LOG(WARNING) << "Failed to get JNIEnv";
+ return st;
}
env->CallNonvirtualVoidMethod(executor_obj, executor_cl, executor_close_id);
- Status st = JniUtil::GetJniExceptionMsg(env);
+ st = JniUtil::GetJniExceptionMsg(env);
if (!st.ok()) {
LOG(WARNING) << "Failed to close JAVA UDAF: " << st.to_string();
+ return st;
}
- env->DeleteGlobalRef(executor_cl);
- env->DeleteGlobalRef(executor_obj);
+ return Status::OK();
}
Status init_udaf(const TFunction& fn, const std::string& local_location) {
@@ -268,8 +278,8 @@ public:
}
void create(AggregateDataPtr __restrict place) const override {
+ new (place) Data(argument_types.size());
if (_first_created) {
- new (place) Data(argument_types.size());
Status status = Status::OK();
SAFE_CREATE(RETURN_IF_STATUS_ERROR(status,
this->data(place).init_udaf(_fn, _local_location)),
@@ -279,16 +289,24 @@ public:
});
_first_created = false;
_exec_place = place;
+ if (UNLIKELY(!status.ok())) {
+ throw doris::Exception(ErrorCode::INTERNAL_ERROR, status.to_string());
+ }
}
}
// To avoid multiple times JNI call, Here will destroy all data at once
void destroy(AggregateDataPtr __restrict place) const noexcept override {
if (place == _exec_place) {
- static_cast<void>(this->data(_exec_place).destroy());
- this->data(_exec_place).~Data();
+ Status status = Status::OK();
+ status = this->data(_exec_place).destroy();
+ status = this->data(_exec_place).close_and_delete_object();
_first_created = true;
+ if (UNLIKELY(!status.ok())) {
+ LOG(WARNING) << "Failed to destroy function: " << status.to_string();
+ }
}
+ this->data(place).~Data();
}
String get_name() const override { return _fn.name.function_name; }
@@ -372,7 +390,6 @@ public:
// so it's can't call ~Data, only to change _destory_deserialize flag.
void deserialize(AggregateDataPtr __restrict place, BufferReadable& buf,
Arena*) const override {
- new (place) Data(argument_types.size());
this->data(place).read(buf);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 11/15: [enhance](auth)enable col auth (#32659)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1d4e5a1c58fa15de9f0ae4182c72848c9f4a9e86
Author: zhangdong <49...@qq.com>
AuthorDate: Sat Mar 23 06:37:32 2024 +0800
[enhance](auth)enable col auth (#32659)
---
fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 5 -----
fe/fe-core/src/main/java/org/apache/doris/analysis/GrantStmt.java | 3 +--
.../src/test/java/org/apache/doris/mysql/privilege/AuthTest.java | 2 --
3 files changed, 1 insertion(+), 9 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 5fd9a969bc5..228acd5e88d 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2268,11 +2268,6 @@ public class Config extends ConfigBase {
"Whether to use mysql's bigint type to return Doris's largeint type"})
public static boolean use_mysql_bigint_for_largeint = false;
- @ConfField(description = {
- "是否开启列权限",
- "Whether to enable col auth"})
- public static boolean enable_col_auth = false;
-
@ConfField
public static boolean forbid_running_alter_job = false;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/GrantStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/GrantStmt.java
index eb8ce67fd69..6712eec124a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/GrantStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/GrantStmt.java
@@ -20,7 +20,6 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.AccessPrivilegeWithCols;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Config;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.FeNameFormat;
@@ -178,7 +177,7 @@ public class GrantStmt extends DdlStmt {
public static void checkAccessPrivileges(
List<AccessPrivilegeWithCols> accessPrivileges) throws AnalysisException {
for (AccessPrivilegeWithCols access : accessPrivileges) {
- if ((!access.getAccessPrivilege().canHasColPriv() || !Config.enable_col_auth) && !CollectionUtils
+ if ((!access.getAccessPrivilege().canHasColPriv()) && !CollectionUtils
.isEmpty(access.getCols())) {
throw new AnalysisException(
String.format("%s do not support col auth.", access.getAccessPrivilege().name()));
diff --git a/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/AuthTest.java b/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/AuthTest.java
index 8f1dde6eac7..0bbc84b5b1a 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/AuthTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/AuthTest.java
@@ -34,7 +34,6 @@ import org.apache.doris.catalog.AccessPrivilegeWithCols;
import org.apache.doris.catalog.DomainResolver;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.UserException;
@@ -151,7 +150,6 @@ public class AuthTest {
};
resolver = new MockDomainResolver(auth);
- Config.enable_col_auth = true;
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 10/15: [Improve](build-script)Add be-extension-ignore to ignore avro-scanner when building release packages (#32699)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 33e40345b6c80d4ecf4468e30cfb432e3893d28b
Author: wudongliang <46...@users.noreply.github.com>
AuthorDate: Fri Mar 22 22:54:19 2024 +0800
[Improve](build-script)Add be-extension-ignore to ignore avro-scanner when building release packages (#32699)
---
build-for-release.sh | 2 +-
build.sh | 36 ++++++++++++++++++++++++++++++++++++
2 files changed, 37 insertions(+), 1 deletion(-)
diff --git a/build-for-release.sh b/build-for-release.sh
index 2a7569aaa30..bdf95b76b56 100755
--- a/build-for-release.sh
+++ b/build-for-release.sh
@@ -145,7 +145,7 @@ echo "JAR: ${OUTPUT_EXT}"
sh build.sh --clean &&
USE_AVX2="${_USE_AVX2}" sh build.sh &&
- USE_AVX2="${_USE_AVX2}" sh build.sh --be --meta-tool
+ USE_AVX2="${_USE_AVX2}" sh build.sh --be --meta-tool --be-extension-ignore avro-scanner
echo "Begin to pack"
rm -rf "${OUTPUT}"
diff --git a/build.sh b/build.sh
index 443b0b380af..00e6e4d70e6 100755
--- a/build.sh
+++ b/build.sh
@@ -48,6 +48,7 @@ Usage: $0 <options>
--spark-dpp build Spark DPP application. Default ON.
--hive-udf build Hive UDF library for Spark Load. Default ON.
--be-java-extensions build Backend java extensions. Default ON.
+ --be-extension-ignore build be-java-extensions package, choose which modules to ignore. Multiple modules separated by commas.
--clean clean and build target
--output specify the output directory
-j build Backend parallel
@@ -69,6 +70,7 @@ Usage: $0 <options>
$0 --be --fe build Backend, Frontend, Spark Dpp application and Java UDF library
$0 --be --coverage build Backend with coverage enabled
$0 --be --output PATH build Backend, the result will be output to PATH(relative paths are available)
+ $0 --be-extension-ignore avro-scanner build be-java-extensions, choose which modules to ignore. Multiple modules separated by commas, like --be-extension-ignore avro-scanner,hudi-scanner
USE_AVX2=0 $0 --be build Backend and not using AVX2 instruction.
USE_AVX2=0 STRIP_DEBUG_INFO=ON $0 build all and not using AVX2 instruction, and strip the debug info for Backend
@@ -124,6 +126,7 @@ if ! OPTS="$(getopt \
-l 'spark-dpp' \
-l 'hive-udf' \
-l 'be-java-extensions' \
+ -l 'be-extension-ignore:' \
-l 'clean' \
-l 'coverage' \
-l 'help' \
@@ -231,6 +234,10 @@ else
DORIS_OUTPUT="$2"
shift 2
;;
+ --be-extension-ignore)
+ BE_EXTENSION_IGNORE="$2"
+ shift 2
+ ;;
--)
shift
break
@@ -489,6 +496,14 @@ if [[ "${BUILD_BE_JAVA_EXTENSIONS}" -eq 1 ]]; then
modules+=("be-java-extensions/max-compute-scanner")
modules+=("be-java-extensions/avro-scanner")
modules+=("be-java-extensions/preload-extensions")
+
+ # If the BE_EXTENSION_IGNORE variable is not empty, remove the modules that need to be ignored from FE_MODULES
+ if [[ -n "${BE_EXTENSION_IGNORE}" ]]; then
+ IFS=',' read -r -a ignore_modules <<<"${BE_EXTENSION_IGNORE}"
+ for module in "${ignore_modules[@]}"; do
+ modules=("${modules[@]/be-java-extensions\/${module}/}")
+ done
+ fi
fi
FE_MODULES="$(
IFS=','
@@ -728,6 +743,27 @@ EOF
extensions_modules+=("avro-scanner")
extensions_modules+=("preload-extensions")
+ if [[ -n "${BE_EXTENSION_IGNORE}" ]]; then
+ IFS=',' read -r -a ignore_modules <<<"${BE_EXTENSION_IGNORE}"
+ new_modules=()
+ for module in "${extensions_modules[@]}"; do
+ module=${module// /}
+ if [[ -n "${module}" ]]; then
+ ignore=0
+ for ignore_module in "${ignore_modules[@]}"; do
+ if [[ "${module}" == "${ignore_module}" ]]; then
+ ignore=1
+ break
+ fi
+ done
+ if [[ "${ignore}" -eq 0 ]]; then
+ new_modules+=("${module}")
+ fi
+ fi
+ done
+ extensions_modules=("${new_modules[@]}")
+ fi
+
BE_JAVA_EXTENSIONS_DIR="${DORIS_OUTPUT}/be/lib/java_extensions/"
rm -rf "${BE_JAVA_EXTENSIONS_DIR}"
mkdir "${BE_JAVA_EXTENSIONS_DIR}"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 12/15: [conf](mysql) opt mysql network timeout to 600s #32545
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 5077f7dd9dd24c7a3ce5606561c8db5d35b48067
Author: camby <ca...@tencent.com>
AuthorDate: Sat Mar 23 06:55:57 2024 +0800
[conf](mysql) opt mysql network timeout to 600s #32545
---
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 7abe546d53b..d649f442f5f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -712,11 +712,11 @@ public class SessionVariable implements Serializable, Writable {
// The number of seconds to wait for a block to be written to a connection before aborting the write
@VariableMgr.VarAttr(name = NET_WRITE_TIMEOUT)
- public int netWriteTimeout = 60;
+ public int netWriteTimeout = 600;
// The number of seconds to wait for a block to be written to a connection before aborting the write
@VariableMgr.VarAttr(name = NET_READ_TIMEOUT)
- public int netReadTimeout = 60;
+ public int netReadTimeout = 600;
// The current time zone
@VariableMgr.VarAttr(name = TIME_ZONE, needForward = true)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 02/15: [refactor](nereids)unify outputTupleDesc and projection (#32093)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 8b960beaec8140f0613917b02d808a42f2814784
Author: minghong <en...@gmail.com>
AuthorDate: Fri Mar 22 18:34:19 2024 +0800
[refactor](nereids)unify outputTupleDesc and projection (#32093)
* unify join node output project
---
.../apache/doris/job/extensions/mtmv/MTMVTask.java | 1 +
.../glue/translator/PhysicalPlanTranslator.java | 14 ++--
.../org/apache/doris/planner/HashJoinNode.java | 34 +++++-----
.../org/apache/doris/planner/JoinNodeBase.java | 78 +++++++++++-----------
.../apache/doris/planner/NestedLoopJoinNode.java | 30 ++++-----
.../java/org/apache/doris/planner/PlanNode.java | 16 +++--
.../table_function/explode_json_array.groovy | 32 +++++----
7 files changed, 102 insertions(+), 103 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
index d7722e090d0..499d1571d39 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
@@ -222,6 +222,7 @@ public class MTMVTask extends AbstractTask {
executor = new StmtExecutor(ctx, new LogicalPlanAdapter(command, ctx.getStatementContext()));
ctx.setExecutor(executor);
ctx.setQueryId(queryId);
+ ctx.getState().setNereids(true);
command.run(ctx, executor);
if (ctx.getState().getStateType() != MysqlStateType.OK) {
throw new JobException(ctx.getState().getErrorMessage());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 802c39ca838..c489c06b548 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -1323,6 +1323,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
null, null, null, hashJoin.isMarkJoin());
hashJoinNode.setNereidsId(hashJoin.getId());
hashJoinNode.setChildrenDistributeExprLists(distributeExprLists);
+ hashJoinNode.setUseSpecificProjections(false);
PlanFragment currentFragment = connectJoinNode(hashJoinNode, leftFragment, rightFragment, context, hashJoin);
if (JoinUtils.shouldColocateJoin(physicalHashJoin)) {
@@ -1550,8 +1551,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
TupleDescriptor outputDescriptor = context.generateTupleDesc();
outputSlotReferences.forEach(s -> context.createSlotDesc(outputDescriptor, s));
- hashJoinNode.setvOutputTupleDesc(outputDescriptor);
- hashJoinNode.setvSrcToOutputSMap(srcToOutput);
+ hashJoinNode.setOutputTupleDesc(outputDescriptor);
+ hashJoinNode.setProjectList(srcToOutput);
}
if (hashJoin.getStats() != null) {
hashJoinNode.setCardinality((long) hashJoin.getStats().getRowCount());
@@ -1585,6 +1586,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
NestedLoopJoinNode nestedLoopJoinNode = new NestedLoopJoinNode(context.nextPlanNodeId(),
leftFragmentPlanRoot, rightFragmentPlanRoot, tupleIds, JoinType.toJoinOperator(joinType),
null, null, null, nestedLoopJoin.isMarkJoin());
+ nestedLoopJoinNode.setUseSpecificProjections(false);
nestedLoopJoinNode.setNereidsId(nestedLoopJoin.getId());
nestedLoopJoinNode.setChildrenDistributeExprLists(distributeExprLists);
if (nestedLoopJoin.getStats() != null) {
@@ -1731,8 +1733,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
TupleDescriptor outputDescriptor = context.generateTupleDesc();
outputSlotReferences.forEach(s -> context.createSlotDesc(outputDescriptor, s));
- nestedLoopJoinNode.setvOutputTupleDesc(outputDescriptor);
- nestedLoopJoinNode.setvSrcToOutputSMap(srcToOutput);
+ nestedLoopJoinNode.setOutputTupleDesc(outputDescriptor);
+ nestedLoopJoinNode.setProjectList(srcToOutput);
}
if (nestedLoopJoin.getStats() != null) {
nestedLoopJoinNode.setCardinality((long) nestedLoopJoin.getStats().getRowCount());
@@ -1858,8 +1860,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
if (inputPlanNode instanceof JoinNodeBase) {
TupleDescriptor tupleDescriptor = generateTupleDesc(slots, null, context);
JoinNodeBase joinNode = (JoinNodeBase) inputPlanNode;
- joinNode.setvOutputTupleDesc(tupleDescriptor);
- joinNode.setvSrcToOutputSMap(projectionExprs);
+ joinNode.setOutputTupleDesc(tupleDescriptor);
+ joinNode.setProjectList(projectionExprs);
// prune the hashOutputSlotIds
if (joinNode instanceof HashJoinNode) {
((HashJoinNode) joinNode).getHashOutputSlotIds().clear();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
index d8cc4a77a0a..c3cbf2afce1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
@@ -171,7 +171,7 @@ public class HashJoinNode extends JoinNodeBase {
nullableTupleIds.addAll(outer.getTupleIds());
}
vIntermediateTupleDescList = Lists.newArrayList(intermediateTuple);
- vOutputTupleDesc = outputTuple;
+ this.outputTupleDesc = outputTuple;
vSrcToOutputSMap = new ExprSubstitutionMap(srcToOutputList, Collections.emptyList());
}
@@ -220,7 +220,7 @@ public class HashJoinNode extends JoinNodeBase {
nullableTupleIds.addAll(outer.getTupleIds());
}
vIntermediateTupleDescList = Lists.newArrayList(intermediateTuple);
- vOutputTupleDesc = outputTuple;
+ this.outputTupleDesc = outputTuple;
vSrcToOutputSMap = new ExprSubstitutionMap(srcToOutputList, Collections.emptyList());
}
@@ -800,26 +800,24 @@ public class HashJoinNode extends JoinNodeBase {
msg.hash_join_node.addToHashOutputSlotIds(slotId.asInt());
}
}
- if (vSrcToOutputSMap != null) {
- for (int i = 0; i < vSrcToOutputSMap.size(); i++) {
- // TODO: Enable it after we support new optimizers
- // if (ConnectContext.get().getSessionVariable().isEnableNereidsPlanner()) {
- // msg.addToProjections(vSrcToOutputSMap.getLhs().get(i).treeToThrift());
- // } else
- msg.hash_join_node.addToSrcExprList(vSrcToOutputSMap.getLhs().get(i).treeToThrift());
+ if (useSpecificProjections) {
+ if (vSrcToOutputSMap != null && vSrcToOutputSMap.getLhs() != null && outputTupleDesc != null) {
+ for (int i = 0; i < vSrcToOutputSMap.size(); i++) {
+ msg.hash_join_node.addToSrcExprList(vSrcToOutputSMap.getLhs().get(i).treeToThrift());
+ }
+ }
+ if (outputTupleDesc != null) {
+ msg.hash_join_node.setVoutputTupleId(outputTupleDesc.getId().asInt());
}
}
- if (vOutputTupleDesc != null) {
- msg.hash_join_node.setVoutputTupleId(vOutputTupleDesc.getId().asInt());
- // TODO Enable it after we support new optimizers
- // msg.setOutputTupleId(vOutputTupleDesc.getId().asInt());
- }
+
if (vIntermediateTupleDescList != null) {
for (TupleDescriptor tupleDescriptor : vIntermediateTupleDescList) {
msg.hash_join_node.addToVintermediateTupleIdList(tupleDescriptor.getId().asInt());
}
}
msg.hash_join_node.setDistType(isColocate ? TJoinDistributionType.COLOCATE : distrMode.toThrift());
+ msg.hash_join_node.setUseSpecificProjections(useSpecificProjections);
}
@Override
@@ -862,9 +860,11 @@ public class HashJoinNode extends JoinNodeBase {
output.append(getRuntimeFilterExplainString(true));
}
output.append(detailPrefix).append(String.format("cardinality=%,d", cardinality)).append("\n");
- // todo unify in plan node
- if (vOutputTupleDesc != null) {
- output.append(detailPrefix).append("vec output tuple id: ").append(vOutputTupleDesc.getId()).append("\n");
+ if (outputTupleDesc != null) {
+ output.append(detailPrefix).append("vec output tuple id: ").append(outputTupleDesc.getId()).append("\n");
+ }
+ if (outputTupleDesc != null) {
+ output.append(detailPrefix).append("output tuple id: ").append(outputTupleDesc.getId()).append("\n");
}
if (vIntermediateTupleDescList != null) {
output.append(detailPrefix).append("vIntermediate tuple ids: ");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/JoinNodeBase.java b/fe/fe-core/src/main/java/org/apache/doris/planner/JoinNodeBase.java
index b635cfda59d..fbe12b3d6a0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/JoinNodeBase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/JoinNodeBase.java
@@ -44,7 +44,6 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -57,10 +56,14 @@ public abstract class JoinNodeBase extends PlanNode {
protected final TableRef innerRef;
protected final JoinOperator joinOp;
protected final boolean isMark;
- protected TupleDescriptor vOutputTupleDesc;
protected ExprSubstitutionMap vSrcToOutputSMap;
protected List<TupleDescriptor> vIntermediateTupleDescList;
+ // in thrift, every planNode denote its output tupelId and projections by output_tuple_id and projections
+ // but for legacy reasons, JoinNode has its specific representations: voutput_tuple_id and src_expr_list
+ // if useSpecificProjections true, set the output to its specific attributes.
+ protected boolean useSpecificProjections = true;
+
public JoinNodeBase(PlanNodeId id, String planNodeName, StatisticalType statisticalType,
PlanNode outer, PlanNode inner, TableRef innerRef) {
super(id, planNodeName, statisticalType);
@@ -115,7 +118,7 @@ public abstract class JoinNodeBase extends PlanNode {
protected void computeOutputTuple(Analyzer analyzer) throws UserException {
// 1. create new tuple
- vOutputTupleDesc = analyzer.getDescTbl().createTupleDescriptor();
+ outputTupleDesc = analyzer.getDescTbl().createTupleDescriptor();
boolean copyLeft = false;
boolean copyRight = false;
boolean leftNullable = false;
@@ -169,7 +172,7 @@ public abstract class JoinNodeBase extends PlanNode {
getChild(0) instanceof JoinNodeBase && analyzer.isOuterJoined(leftTupleDesc.getId());
for (SlotDescriptor leftSlotDesc : leftTupleDesc.getSlots()) {
SlotDescriptor outputSlotDesc =
- analyzer.getDescTbl().copySlotDescriptor(vOutputTupleDesc, leftSlotDesc);
+ analyzer.getDescTbl().copySlotDescriptor(outputTupleDesc, leftSlotDesc);
if (leftNullable) {
outputSlotDesc.setIsNullable(true);
leftNullableNumber++;
@@ -189,7 +192,7 @@ public abstract class JoinNodeBase extends PlanNode {
getChild(1) instanceof JoinNodeBase && analyzer.isOuterJoined(rightTupleDesc.getId());
for (SlotDescriptor rightSlotDesc : rightTupleDesc.getSlots()) {
SlotDescriptor outputSlotDesc =
- analyzer.getDescTbl().copySlotDescriptor(vOutputTupleDesc, rightSlotDesc);
+ analyzer.getDescTbl().copySlotDescriptor(outputTupleDesc, rightSlotDesc);
if (rightNullable) {
outputSlotDesc.setIsNullable(true);
rightNullableNumber++;
@@ -206,7 +209,7 @@ public abstract class JoinNodeBase extends PlanNode {
if (isMarkJoin() && analyzer.needPopUpMarkTuple(innerRef)) {
SlotDescriptor markSlot = analyzer.getMarkTuple(innerRef).getSlots().get(0);
SlotDescriptor outputSlotDesc =
- analyzer.getDescTbl().copySlotDescriptor(vOutputTupleDesc, markSlot);
+ analyzer.getDescTbl().copySlotDescriptor(outputTupleDesc, markSlot);
srcTblRefToOutputTupleSmap.put(new SlotRef(markSlot), new SlotRef(outputSlotDesc));
}
@@ -224,7 +227,7 @@ public abstract class JoinNodeBase extends PlanNode {
}
}
- vOutputTupleDesc.computeStatAndMemLayout();
+ outputTupleDesc.computeStatAndMemLayout();
// 3. add tupleisnull in null-side
Preconditions.checkState(srcTblRefToOutputTupleSmap.getLhs().size() == vSrcToOutputSMap.getLhs().size());
// Condition1: the left child is null-side
@@ -265,8 +268,8 @@ public abstract class JoinNodeBase extends PlanNode {
public void initOutputSlotIds(Set<SlotId> requiredSlotIdSet, Analyzer analyzer) {
outputSlotIds = Lists.newArrayList();
List<TupleDescriptor> outputTupleDescList = Lists.newArrayList();
- if (vOutputTupleDesc != null) {
- outputTupleDescList.add(vOutputTupleDesc);
+ if (outputTupleDesc != null) {
+ outputTupleDescList.add(outputTupleDesc);
} else {
for (TupleId tupleId : tupleIds) {
outputTupleDescList.add(analyzer.getTupleDesc(tupleId));
@@ -296,13 +299,13 @@ public abstract class JoinNodeBase extends PlanNode {
@Override
public void projectOutputTuple() {
- if (vOutputTupleDesc == null) {
+ if (outputTupleDesc == null) {
return;
}
- if (vOutputTupleDesc.getSlots().size() == outputSlotIds.size()) {
+ if (outputTupleDesc.getSlots().size() == outputSlotIds.size()) {
return;
}
- Iterator<SlotDescriptor> iterator = vOutputTupleDesc.getSlots().iterator();
+ Iterator<SlotDescriptor> iterator = outputTupleDesc.getSlots().iterator();
while (iterator.hasNext()) {
SlotDescriptor slotDescriptor = iterator.next();
boolean keep = false;
@@ -318,7 +321,7 @@ public abstract class JoinNodeBase extends PlanNode {
vSrcToOutputSMap.removeByRhsExpr(slotRef);
}
}
- vOutputTupleDesc.computeStatAndMemLayout();
+ outputTupleDesc.computeStatAndMemLayout();
}
protected abstract Pair<Boolean, Boolean> needToCopyRightAndLeft();
@@ -404,14 +407,14 @@ public abstract class JoinNodeBase extends PlanNode {
// 5. replace tuple is null expr
TupleIsNullPredicate.substitueListForTupleIsNull(vSrcToOutputSMap.getLhs(), originTidsToIntermediateTidMap);
- Preconditions.checkState(vSrcToOutputSMap.getLhs().size() == vOutputTupleDesc.getSlots().size());
+ Preconditions.checkState(vSrcToOutputSMap.getLhs().size() == outputTupleDesc.getSlots().size());
List<Expr> exprs = vSrcToOutputSMap.getLhs();
- ArrayList<SlotDescriptor> slots = vOutputTupleDesc.getSlots();
+ ArrayList<SlotDescriptor> slots = outputTupleDesc.getSlots();
for (int i = 0; i < slots.size(); i++) {
slots.get(i).setIsNullable(exprs.get(i).isNullable());
}
vSrcToOutputSMap.reCalculateNullableInfoForSlotInRhs();
- vOutputTupleDesc.computeMemLayout();
+ outputTupleDesc.computeMemLayout();
}
protected abstract List<SlotId> computeSlotIdsForJoinConjuncts(Analyzer analyzer);
@@ -479,7 +482,7 @@ public abstract class JoinNodeBase extends PlanNode {
/**
* If parent wants to get join node tupleids,
* it will call this function instead of read properties directly.
- * The reason is that the tuple id of vOutputTupleDesc the real output tuple id for join node.
+ * The reason is that the tuple id of outputTupleDesc the real output tuple id for join node.
* <p>
* If you read the properties of @tupleids directly instead of this function,
* it reads the input id of the current node.
@@ -487,16 +490,16 @@ public abstract class JoinNodeBase extends PlanNode {
@Override
public ArrayList<TupleId> getTupleIds() {
Preconditions.checkState(tupleIds != null);
- if (vOutputTupleDesc != null) {
- return Lists.newArrayList(vOutputTupleDesc.getId());
+ if (outputTupleDesc != null) {
+ return Lists.newArrayList(outputTupleDesc.getId());
}
return tupleIds;
}
@Override
public ArrayList<TupleId> getOutputTblRefIds() {
- if (vOutputTupleDesc != null) {
- return Lists.newArrayList(vOutputTupleDesc.getId());
+ if (outputTupleDesc != null) {
+ return Lists.newArrayList(outputTupleDesc.getId());
}
switch (joinOp) {
case LEFT_SEMI_JOIN:
@@ -513,8 +516,8 @@ public abstract class JoinNodeBase extends PlanNode {
@Override
public List<TupleId> getOutputTupleIds() {
- if (vOutputTupleDesc != null) {
- return Lists.newArrayList(vOutputTupleDesc.getId());
+ if (outputTupleDesc != null) {
+ return Lists.newArrayList(outputTupleDesc.getId());
}
switch (joinOp) {
case LEFT_SEMI_JOIN:
@@ -544,13 +547,6 @@ public abstract class JoinNodeBase extends PlanNode {
return Math.max(children.get(0).getNumInstances(), children.get(1).getNumInstances());
}
- /**
- * Used by nereids.
- */
- public void setvOutputTupleDesc(TupleDescriptor vOutputTupleDesc) {
- this.vOutputTupleDesc = vOutputTupleDesc;
- }
-
/**
* Used by nereids.
*/
@@ -558,13 +554,6 @@ public abstract class JoinNodeBase extends PlanNode {
this.vIntermediateTupleDescList = vIntermediateTupleDescList;
}
- /**
- * Used by nereids.
- */
- public void setvSrcToOutputSMap(List<Expr> lhs) {
- this.vSrcToOutputSMap = new ExprSubstitutionMap(lhs, Collections.emptyList());
- }
-
public void setOutputSmap(ExprSubstitutionMap smap, Analyzer analyzer) {
outputSmap = smap;
ExprSubstitutionMap tmpSmap = new ExprSubstitutionMap(Lists.newArrayList(vSrcToOutputSMap.getRhs()),
@@ -572,12 +561,12 @@ public abstract class JoinNodeBase extends PlanNode {
List<Expr> newRhs = Lists.newArrayList();
boolean bSmapChanged = false;
for (Expr rhsExpr : smap.getRhs()) {
- if (rhsExpr instanceof SlotRef || !rhsExpr.isBound(vOutputTupleDesc.getId())) {
+ if (rhsExpr instanceof SlotRef || !rhsExpr.isBound(outputTupleDesc.getId())) {
newRhs.add(rhsExpr);
} else {
// we need do project in the join node
// add a new slot for projection result and add the project expr to vSrcToOutputSMap
- SlotDescriptor slotDesc = analyzer.addSlotDescriptor(vOutputTupleDesc);
+ SlotDescriptor slotDesc = analyzer.addSlotDescriptor(outputTupleDesc);
slotDesc.initFromExpr(rhsExpr);
slotDesc.setIsMaterialized(true);
// the project expr is from smap, which use the slots of hash join node's output tuple
@@ -594,7 +583,16 @@ public abstract class JoinNodeBase extends PlanNode {
if (bSmapChanged) {
outputSmap.updateRhsExprs(newRhs);
- vOutputTupleDesc.computeStatAndMemLayout();
+ outputTupleDesc.computeStatAndMemLayout();
}
}
+
+ public void setUseSpecificProjections(boolean useSpecificProjections) {
+ this.useSpecificProjections = useSpecificProjections;
+ }
+
+
+ public boolean isUseSpecificProjections() {
+ return useSpecificProjections;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java
index 9feac7e79ff..983cbfd5884 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java
@@ -127,7 +127,7 @@ public class NestedLoopJoinNode extends JoinNodeBase {
nullableTupleIds.addAll(outer.getTupleIds());
}
vIntermediateTupleDescList = Lists.newArrayList(intermediateTuple);
- vOutputTupleDesc = outputTuple;
+ outputTupleDesc = outputTuple;
vSrcToOutputSMap = new ExprSubstitutionMap(srcToOutputList, Collections.emptyList());
}
@@ -186,26 +186,25 @@ public class NestedLoopJoinNode extends JoinNodeBase {
}
msg.nested_loop_join_node.setIsMark(isMarkJoin());
- if (vSrcToOutputSMap != null) {
- for (int i = 0; i < vSrcToOutputSMap.size(); i++) {
- // TODO: Enable it after we support new optimizers
- // if (ConnectContext.get().getSessionVariable().isEnableNereidsPlanner()) {
- // msg.addToProjections(vSrcToOutputSMap.getLhs().get(i).treeToThrift());
- // } else
- msg.nested_loop_join_node.addToSrcExprList(vSrcToOutputSMap.getLhs().get(i).treeToThrift());
+
+ if (useSpecificProjections) {
+ if (vSrcToOutputSMap != null && vSrcToOutputSMap.getLhs() != null && outputTupleDesc != null) {
+ for (int i = 0; i < vSrcToOutputSMap.size(); i++) {
+ msg.nested_loop_join_node.addToSrcExprList(vSrcToOutputSMap.getLhs().get(i).treeToThrift());
+ }
+ }
+ if (outputTupleDesc != null) {
+ msg.nested_loop_join_node.setVoutputTupleId(outputTupleDesc.getId().asInt());
}
}
- if (vOutputTupleDesc != null) {
- msg.nested_loop_join_node.setVoutputTupleId(vOutputTupleDesc.getId().asInt());
- // TODO Enable it after we support new optimizers
- // msg.setOutputTupleId(vOutputTupleDesc.getId().asInt());
- }
+
if (vIntermediateTupleDescList != null) {
for (TupleDescriptor tupleDescriptor : vIntermediateTupleDescList) {
msg.nested_loop_join_node.addToVintermediateTupleIdList(tupleDescriptor.getId().asInt());
}
}
msg.nested_loop_join_node.setIsOutputLeftSideOnly(isOutputLeftSideOnly);
+ msg.nested_loop_join_node.setUseSpecificProjections(useSpecificProjections);
msg.node_type = TPlanNodeType.CROSS_JOIN_NODE;
}
@@ -258,10 +257,7 @@ public class NestedLoopJoinNode extends JoinNodeBase {
}
output.append(detailPrefix).append("is output left side only: ").append(isOutputLeftSideOnly).append("\n");
output.append(detailPrefix).append(String.format("cardinality=%,d", cardinality)).append("\n");
- // todo unify in plan node
- if (vOutputTupleDesc != null) {
- output.append(detailPrefix).append("vec output tuple id: ").append(vOutputTupleDesc.getId()).append("\n");
- }
+
if (vIntermediateTupleDescList != null) {
output.append(detailPrefix).append("vIntermediate tuple ids: ");
for (TupleDescriptor tupleDescriptor : vIntermediateTupleDescList) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
index ee8965c18a5..b404bc4ad35 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
@@ -648,13 +648,17 @@ public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats {
}
toThrift(msg);
container.addToNodes(msg);
- if (projectList != null) {
- for (Expr expr : projectList) {
- msg.addToProjections(expr.treeToThrift());
+
+ // legacy planner set outputTuple and projections inside join node
+ if (!(this instanceof JoinNodeBase) || !(((JoinNodeBase) this).isUseSpecificProjections())) {
+ if (outputTupleDesc != null) {
+ msg.setOutputTupleId(outputTupleDesc.getId().asInt());
+ }
+ if (projectList != null) {
+ for (Expr expr : projectList) {
+ msg.addToProjections(expr.treeToThrift());
+ }
}
- }
- if (outputTupleDesc != null) {
- msg.setOutputTupleId(outputTupleDesc.getId().asInt());
}
if (this instanceof ExchangeNode) {
msg.num_children = 0;
diff --git a/regression-test/suites/query_p0/sql_functions/table_function/explode_json_array.groovy b/regression-test/suites/query_p0/sql_functions/table_function/explode_json_array.groovy
index e4cbe3feda0..e6ed1b62a24 100644
--- a/regression-test/suites/query_p0/sql_functions/table_function/explode_json_array.groovy
+++ b/regression-test/suites/query_p0/sql_functions/table_function/explode_json_array.groovy
@@ -20,71 +20,69 @@
// and modified by Doris.
suite("explode_json_array") {
- def tableName = "person"
-
- sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """ DROP TABLE IF EXISTS person """
sql """
- CREATE TABLE IF NOT EXISTS ${tableName}
+ CREATE TABLE IF NOT EXISTS person
(id INT, name STRING, age INT, class INT, address STRING)
UNIQUE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 8
PROPERTIES("replication_num" = "1")
"""
- sql """ INSERT INTO ${tableName} VALUES
+ sql """ INSERT INTO person VALUES
(100, 'John', 30, 1, 'Street 1'),
(200, 'Mary', NULL, 1, 'Street 2'),
(300, 'Mike', 80, 3, 'Street 3'),
(400, 'Dan', 50, 4, 'Street 4') """
- qt_explode_json_array7 """ SELECT id, name, age, class, address, d_age, c_age FROM ${tableName}
+ qt_explode_json_array7 """ SELECT id, name, age, class, address, d_age, c_age FROM person
LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[30, 60]') t1 as c_age
LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[40, 80]') t2 as d_age
ORDER BY id, c_age, d_age """
- qt_explode_json_array8 """ SELECT c_age, COUNT(1) FROM ${tableName}
+ qt_explode_json_array8 """ SELECT c_age, COUNT(1) FROM person
LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[30, 60]') t1 as c_age
LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[40, 80]') t2 as d_age
GROUP BY c_age ORDER BY c_age """
- qt_explode_json_array_8_invalid """ SELECT c_age, COUNT(1) FROM ${tableName}
+ qt_explode_json_array_8_invalid """ SELECT c_age, COUNT(1) FROM person
LATERAL VIEW EXPLODE_JSON_ARRAY_INT('["1", 60]') t1 as c_age
LATERAL VIEW EXPLODE_JSON_ARRAY_INT('["b", "c"]') t2 as d_age
GROUP BY c_age ORDER BY c_age """
- qt_explode_json_array9 """ SELECT * FROM ${tableName}
+ qt_explode_json_array9 """ SELECT * FROM person
LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[]') t1 AS c_age
ORDER BY id, c_age """
- qt_explode_json_array10 """ SELECT id, name, age, class, address, d, c FROM ${tableName}
+ qt_explode_json_array10 """ SELECT id, name, age, class, address, d, c FROM person
LATERAL VIEW EXPLODE_JSON_ARRAY_STRING('[1, "b", -3]') t1 as c
LATERAL VIEW EXPLODE_JSON_ARRAY_DOUBLE('[1.23, 22.214, 214.1]') t2 as d
ORDER BY id, c, d """
qt_outer_join_explode_json_array11 """SELECT id, age, e1 FROM (SELECT id, age, e1 FROM (SELECT b.id, a.age FROM
- ${tableName} a LEFT JOIN ${tableName} b ON a.id=b.age)T LATERAL VIEW EXPLODE_JSON_ARRAY_STRING('[1, "b", 3]')
+ person a LEFT JOIN person b ON a.id=b.age)T LATERAL VIEW EXPLODE_JSON_ARRAY_STRING('[1, "b", 3]')
TMP AS e1) AS T ORDER BY age, e1"""
qt_outer_join_explode_json_array11 """SELECT id, age, e1 FROM (SELECT id, age, e1 FROM (SELECT b.id, a.age FROM
- ${tableName} a LEFT JOIN ${tableName} b ON a.id=b.age)T LATERAL VIEW EXPLODE_JSON_ARRAY_JSON('[{"id":1,"name":"John"},{"id":2,"name":"Mary"},{"id":3,"name":"Bob"}]')
+ person a LEFT JOIN person b ON a.id=b.age)T LATERAL VIEW EXPLODE_JSON_ARRAY_JSON('[{"id":1,"name":"John"},{"id":2,"name":"Mary"},{"id":3,"name":"Bob"}]')
TMP AS e1) AS T ORDER BY age, e1"""
- qt_explode_json_array12 """ SELECT c_age, COUNT(1) FROM ${tableName}
+ qt_explode_json_array12 """ SELECT c_age, COUNT(1) FROM person
LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[9223372036854775807,9223372036854775808]') t1 as c_age
GROUP BY c_age ORDER BY c_age """
- qt_explode_json_array13 """ SELECT c_age, COUNT(1) FROM ${tableName}
+ qt_explode_json_array13 """ SELECT c_age, COUNT(1) FROM person
LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[-92233720368547758071,-92233720368547758081]') t1 as c_age
GROUP BY c_age ORDER BY c_age """
- qt_explode_json_array14 """ SELECT id, name, age, class, address, d, c FROM ${tableName}
+ qt_explode_json_array14 """ SELECT id, name, age, class, address, d, c FROM person
LATERAL VIEW EXPLODE_JSON_ARRAY_STRING('[1182381637816312, "b", -1273982982312333]') t1 as c
LATERAL VIEW EXPLODE_JSON_ARRAY_DOUBLE('[1.23, 22.214, 214.1]') t2 as d
ORDER BY id, c, d """
- qt_explode_json_array15 """ SELECT id, name, age, class, address, d, c FROM ${tableName}
+ qt_explode_json_array15 """ SELECT id, name, age, class, address, d, c FROM person
LATERAL VIEW EXPLODE_JSON_ARRAY_STRING('[true, "b", false]') t1 as c
LATERAL VIEW EXPLODE_JSON_ARRAY_DOUBLE('[1.23, 22.214, 214.1]') t2 as d
ORDER BY id, c, d """
- qt_explode_json_array16 """ SELECT id, name, age, class, address, d, c FROM ${tableName}
+ qt_explode_json_array16 """ SELECT id, name, age, class, address, d, c FROM person
LATERAL VIEW EXPLODE_JSON_ARRAY_STRING('[null, "b", null]') t1 as c
LATERAL VIEW EXPLODE_JSON_ARRAY_DOUBLE('[1.23, 22.214, 214.1]') t2 as d
ORDER BY id, c, d """
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 03/15: [Fix](Test)Reduce the Sleep time and ensure that the corresponding thread resources are released after the case is executed. (#32665)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit c223d9e7d008c063b36e9b2bd4be1acb471f06a5
Author: Calvin Kirs <ki...@apache.org>
AuthorDate: Fri Mar 22 19:15:47 2024 +0800
[Fix](Test)Reduce the Sleep time and ensure that the corresponding thread resources are released after the case is executed. (#32665)
* [Fix](Test)Reduce the Sleep time and ensure that the corresponding thread resources are released after the case is executed.
* Fix load job id error
---
.../commands/insert/AbstractInsertExecutor.java | 1 +
.../suites/job_p0/test_base_insert_job.groovy | 286 +++++++++++++++++++++
2 files changed, 287 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java
index 922dae6d40b..a9b3ae6d632 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java
@@ -107,6 +107,7 @@ public abstract class AbstractInsertExecutor {
protected final void execImpl(StmtExecutor executor, long jobId) throws Exception {
String queryId = DebugUtil.printId(ctx.queryId());
+ this.jobId = jobId;
coordinator.setLoadZeroTolerance(ctx.getSessionVariable().getEnableInsertStrict());
coordinator.setQueryType(TQueryType.LOAD);
executor.getProfile().addExecutionProfile(coordinator.getExecutionProfile());
diff --git a/regression-test/suites/job_p0/test_base_insert_job.groovy b/regression-test/suites/job_p0/test_base_insert_job.groovy
new file mode 100644
index 00000000000..8fe7c0e7749
--- /dev/null
+++ b/regression-test/suites/job_p0/test_base_insert_job.groovy
@@ -0,0 +1,286 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.Instant;
+import java.time.ZoneId;
+
+suite("test_base_insert_job") {
+ def tableName = "t_test_BASE_inSert_job"
+ def jobName = "insert_recovery_test_base_insert_job"
+ def jobMixedName = "Insert_recovery_Test_base_insert_job"
+ sql """drop table if exists `${tableName}` force"""
+ sql """
+ DROP JOB IF EXISTS where jobname = '${jobName}'
+ """
+ sql """
+ DROP JOB IF EXISTS where jobname = 'JOB'
+ """
+ sql """
+ DROP JOB IF EXISTS where jobname = 'DO'
+ """
+ sql """
+ DROP JOB IF EXISTS where jobname = 'AT'
+ """
+ sql """
+ DROP JOB IF EXISTS where jobname = 'SCHEDULE'
+ """
+ sql """
+ DROP JOB IF EXISTS where jobname = 'STARTS'
+ """
+ sql """
+ DROP JOB IF EXISTS where jobname = 'ENDS'
+ """
+ sql """
+ DROP JOB IF EXISTS where jobname = '${jobMixedName}'
+ """
+
+ sql """
+ DROP JOB IF EXISTS where jobname = '${jobName}'
+ """
+
+ sql """
+ CREATE TABLE IF NOT EXISTS `${tableName}`
+ (
+ `timestamp` DATE NOT NULL COMMENT "['0000-01-01', '9999-12-31']",
+ `type` TINYINT NOT NULL COMMENT "[-128, 127]",
+ `user_id` BIGINT COMMENT "[-9223372036854775808, 9223372036854775807]"
+ )
+ DUPLICATE KEY(`timestamp`, `type`)
+ DISTRIBUTED BY HASH(`type`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+ sql """
+ CREATE JOB ${jobName} ON SCHEDULE every 1 second comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
+ """
+ Thread.sleep(2500)
+ sql """
+ PAUSE JOB where jobname = '${jobName}'
+ """
+ def tblDatas = sql """select * from ${tableName}"""
+ println tblDatas
+ assert 3 >= tblDatas.size() >= (2 as Boolean) //at least 2 records, some times 3 records
+ def pauseJobId = sql """select id from jobs("type"="insert") where Name='${jobName}'"""
+ def taskStatus = sql """select status from tasks("type"="insert") where jobid= '${pauseJobId.get(0).get(0)}'"""
+ println taskStatus
+ for (int i = 0; i < taskStatus.size(); i++) {
+ assert taskStatus.get(i).get(0) != "FAILED"||taskStatus.get(i).get(0) != "STOPPED"||taskStatus.get(i).get(0) != "STOPPED"
+ }
+ sql """
+ CREATE JOB ${jobMixedName} ON SCHEDULE every 1 second DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
+ """
+ def mixedNameJobs = sql """select name,comment from jobs("type"="insert") where Name='${jobMixedName}'"""
+ println mixedNameJobs
+ assert mixedNameJobs.size() == 1 && mixedNameJobs.get(0).get(0) == jobMixedName
+ assert mixedNameJobs.get(0).get(1) == ''
+ sql """
+ DROP JOB IF EXISTS where jobname = '${jobName}'
+ """
+ sql """
+ DROP JOB IF EXISTS where jobname = '${jobMixedName}'
+ """
+
+ sql """drop table if exists `${tableName}` force """
+ sql """
+ CREATE TABLE IF NOT EXISTS `${tableName}`
+ (
+ `timestamp` DATE NOT NULL COMMENT "['0000-01-01', '9999-12-31']",
+ `type` TINYINT NOT NULL COMMENT "[-128, 127]",
+ `user_id` BIGINT COMMENT "[-9223372036854775808, 9223372036854775807]"
+ )
+ DUPLICATE KEY(`timestamp`, `type`)
+ DISTRIBUTED BY HASH(`type`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+ def dataCount = sql """select count(*) from ${tableName}"""
+ assert dataCount.get(0).get(0) == 0
+ sql """
+ CREATE JOB ${jobName} ON SCHEDULE at current_timestamp comment 'test for test&68686781jbjbhj//ncsa' DO insert into ${tableName} values ('2023-07-19', 2, 1001);
+ """
+
+ Thread.sleep(2000)
+ def onceJob = sql """ select id,ExecuteSql from jobs("type"="insert") where Name like '%${jobName}%' and ExecuteType='ONE_TIME' """
+ assert onceJob.size() == 1
+ def onceJobId = onceJob.get(0).get(0);
+ def onceJobSql = onceJob.get(0).get(1);
+ println onceJobSql
+ // test cancel task
+ def datas = sql """select status,taskid from tasks("type"="insert") where jobid= ${onceJobId}"""
+ println datas
+ assert datas.size() == 1
+ assert datas.get(0).get(0) == "FINISHED"
+ // check table data
+ def dataCount1 = sql """select count(1) from ${tableName}"""
+ assert dataCount1.get(0).get(0) == 1
+ // check job status
+ def oncejob = sql """select status,comment from jobs("type"="insert") where Name='${jobName}' """
+ println oncejob
+ assert oncejob.get(0).get(0) == "FINISHED"
+ //assert comment
+ assert oncejob.get(0).get(1) == "test for test&68686781jbjbhj//ncsa"
+ sql """
+ DROP JOB IF EXISTS where jobname = 'press'
+ """
+
+ sql """
+ CREATE JOB press ON SCHEDULE every 10 hour starts CURRENT_TIMESTAMP comment 'test for test&68686781jbjbhj//ncsa' DO insert into ${tableName} values ('2023-07-19', 99, 99);
+ """
+ Thread.sleep(5000)
+ def pressJob = sql """ select * from jobs("type"="insert") where name='press' """
+ println pressJob
+
+ def recurringTableDatas = sql """ select count(1) from ${tableName} where user_id=99 and type=99 """
+ assert recurringTableDatas.get(0).get(0) == 1
+ sql """
+ DROP JOB IF EXISTS where jobname = '${jobName}'
+ """
+ sql """
+ CREATE JOB ${jobName} ON SCHEDULE every 1 second comment 'test for test&68686781jbjbhj//ncsa' DO insert into ${tableName} values ('2023-07-19',5, 1001);
+ """
+
+ Thread.sleep(2000)
+
+ sql """
+ PAUSE JOB where jobname = '${jobName}'
+ """
+ def job = sql """ select id,ExecuteSql from jobs("type"="insert") where Name like '%${jobName}%' """
+ assert job.size() == 1
+ def jobId = job.get(0).get(0);
+ def tasks = sql """ select status from tasks("type"="insert") where jobid= ${jobId} """
+ sql """
+ RESUME JOB where jobname = '${jobName}'
+ """
+ Thread.sleep(2500)
+ def afterResumeTasks = sql """ select status from tasks("type"="insert") where jobid= ${jobId} """
+ println afterResumeTasks
+ assert afterResumeTasks.size() >tasks.size
+ // assert same job name
+ try {
+ sql """
+ CREATE JOB ${jobName} ON SCHEDULE EVERY 10 second comment 'test for test&68686781jbjbhj//ncsa' DO insert into ${tableName} values ('2023-07-19', 10, 1001);
+ """
+ } catch (Exception e) {
+ assert e.getMessage().contains("job name exist, jobName:insert_recovery_test_base_insert_job")
+ }
+ // assert not support stmt
+ try {
+ sql """
+ CREATE JOB ${jobName} ON SCHEDULE at current_timestamp comment 'test' DO update ${tableName} set type=2 where type=1;
+ """
+ } catch (Exception e) {
+ assert e.getMessage().contains("Not support UpdateStmt type in job")
+ }
+ // assert start time greater than current time
+ try {
+ sql """
+ CREATE JOB ${jobName} ON SCHEDULE at '2023-11-13 14:18:07' comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
+ """
+ } catch (Exception e) {
+ assert e.getMessage().contains("startTimeMs must be greater than current time")
+ }
+ // assert end time less than start time
+ try {
+ sql """
+ CREATE JOB test_one_time_error_starts ON SCHEDULE at '2023-11-13 14:18:07' comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
+ """
+ } catch (Exception e) {
+ assert e.getMessage().contains("startTimeMs must be greater than current time")
+ }
+ try {
+ sql """
+ CREATE JOB inner_test ON SCHEDULE at '2023-11-13 14:18:07' comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
+ """
+ } catch (Exception e) {
+ assert e.getMessage().contains("job name can not start with inner_")
+ }
+ // assert end time less than start time
+ try {
+ sql """
+ CREATE JOB test_error_starts ON SCHEDULE every 1 second ends '2023-11-13 14:18:07' comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
+ """
+ } catch (Exception e) {
+ assert e.getMessage().contains("end time cannot be less than start time")
+ }
+ // assert interval time unit can not be years
+ try {
+ sql """
+ CREATE JOB test_error_starts ON SCHEDULE every 1 years ends '2023-11-13 14:18:07' comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
+ """
+ } catch (Exception e) {
+ assert e.getMessage().contains("interval time unit can not be years")
+ }
+
+ // test keyword as job name
+ sql """
+ CREATE JOB JOB ON SCHEDULE every 20 second comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
+ """
+ sql """
+ CREATE JOB SCHEDULE ON SCHEDULE every 20 second comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
+ """
+ sql """
+ CREATE JOB DO ON SCHEDULE every 20 second comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
+ """
+ sql """
+ CREATE JOB AT ON SCHEDULE every 20 second comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
+ """
+
+ sql """
+ CREATE JOB STARTS ON SCHEDULE every 20 second comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
+ """
+
+ sql """
+ CREATE JOB ENDS ON SCHEDULE every 20 second comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
+ """
+
+ def jobCountRsp = sql """select count(1) from jobs("type"="insert") where name in ('JOB','DO','SCHEDULE','AT','STARTS','ENDS')"""
+ assert jobCountRsp.get(0).get(0) == 6
+
+ sql """
+ DROP JOB IF EXISTS where jobname = '${jobName}'
+ """
+ sql """
+ DROP JOB IF EXISTS where jobname = 'JOB'
+ """
+ sql """
+ DROP JOB IF EXISTS where jobname = 'DO'
+ """
+ sql """
+ DROP JOB IF EXISTS where jobname = 'AT'
+ """
+ sql """
+ DROP JOB IF EXISTS where jobname = 'SCHEDULE'
+ """
+ sql """
+ DROP JOB IF EXISTS where jobname = 'STARTS'
+ """
+ sql """
+ DROP JOB IF EXISTS where jobname = 'ENDS'
+ """
+ sql """
+ DROP JOB IF EXISTS where jobname = '${jobMixedName}'
+ """
+
+ sql """
+ DROP JOB IF EXISTS where jobname = '${jobName}'
+ """
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 13/15: [Fix](Export) fix variable name of code #32637
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 5c3fc818bb800f8a759dc73ffa0c038928c4796d
Author: Tiewei Fang <43...@users.noreply.github.com>
AuthorDate: Sat Mar 23 09:15:39 2024 +0800
[Fix](Export) fix variable name of code #32637
---
fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java | 2 +-
fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java | 2 +-
.../org/apache/doris/nereids/trees/plans/commands/ExportCommand.java | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java
index 1a7d9dc93b5..b7a3a84a826 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java
@@ -372,7 +372,7 @@ public class ExportStmt extends StatementBase {
}
this.dataConsistency = ExportJob.CONSISTENT_PARTITION;
} else {
- this.dataConsistency = ExportJob.CONSISTENT_ALL;
+ this.dataConsistency = ExportJob.CONSISTENT_NONE;
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
index fe28151a546..cc47c641b0c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
@@ -109,7 +109,7 @@ public class ExportJob implements Writable {
private static final int MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT = Config.maximum_tablets_of_outfile_in_export;
- public static final String CONSISTENT_ALL = "all";
+ public static final String CONSISTENT_NONE = "none";
public static final String CONSISTENT_PARTITION = "partition";
@SerializedName("id")
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
index e5484336491..b872f399fad 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
@@ -320,7 +320,7 @@ public class ExportCommand extends Command implements ForwardWithSync {
}
exportJob.setDataConsistency(ExportJob.CONSISTENT_PARTITION);
} else {
- exportJob.setDataConsistency(ExportJob.CONSISTENT_ALL);
+ exportJob.setDataConsistency(ExportJob.CONSISTENT_NONE);
}
// Must copy session variable, because session variable may be changed during export job running.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 08/15: [fix](multi-catalog)resolve hive meta store compatibility for different version issues (#32551)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit ac8ba43d8deeda89c678d0a5f57f53ac7774c452
Author: slothever <18...@users.noreply.github.com>
AuthorDate: Fri Mar 22 21:30:43 2024 +0800
[fix](multi-catalog)resolve hive meta store compatibility for different version issues (#32551)
Fix hive list partition at different version.
Only Hive3 uses the prependCatalogToDbName() to wrap db_name.
---
.../hadoop/hive/metastore/HiveMetaStoreClient.java | 98 +++++++++++++---------
1 file changed, 59 insertions(+), 39 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 37bf73b2fa0..b1e47c49484 100644
--- a/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -278,6 +278,16 @@ import javax.security.auth.login.LoginException;
* 7. getSchema()
* 8. getTableColumnStatistics()
* 9. getPartitionColumnStatistics()
+ * 10. getPartitionsByNames()
+ * 11. listPartitions()
+ * 12. alter_partition()
+ * 13. add_partitions()
+ * 14. dropPartition()
+ * 15. alter_table()
+ * 16. alter_table_with_environmentContext()
+ * 17. renamePartition()
+ * 18. truncateTable()
+ * 19. drop_table_with_environment_context()
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
@@ -529,7 +539,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
@Override
public boolean isCompatibleWith(Configuration conf) {
// Make a copy of currentMetaVars, there is a race condition that
- // currentMetaVars might be changed during the execution of the method
+ // currentMetaVars might be changed during the execution of the method
Map<String, String> currentMetaVarsCopy = currentMetaVars;
if (currentMetaVarsCopy == null) {
return false; // recreate
@@ -600,14 +610,14 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
if (hook != null) {
hook.preAlterTable(new_tbl, envContext);
}
- client.alter_table_with_environment_context(prependCatalogToDbName(dbname, conf),
- tbl_name, new_tbl, envContext);
+ client.alter_table_with_environment_context(
+ prependCatalogToDbNameByVersion(hiveVersion, null, dbname, conf), tbl_name, new_tbl, envContext);
}
@Override
public void alter_table(String catName, String dbName, String tblName, Table newTable,
EnvironmentContext envContext) throws TException {
- client.alter_table_with_environment_context(prependCatalogToDbName(catName,
+ client.alter_table_with_environment_context(prependCatalogToDbNameByVersion(hiveVersion, catName,
dbName, conf), tblName, newTable, envContext);
}
@@ -620,7 +630,8 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
@Override
public void renamePartition(String catName, String dbname, String tableName, List<String> part_vals,
Partition newPart) throws TException {
- client.rename_partition(prependCatalogToDbName(catName, dbname, conf), tableName, part_vals, newPart);
+ client.rename_partition(prependCatalogToDbNameByVersion(hiveVersion, catName, dbname, conf),
+ tableName, part_vals, newPart);
}
@@ -882,8 +893,10 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
@Override
public int add_partitions(List<Partition> new_parts) throws TException {
if (new_parts != null && !new_parts.isEmpty() && !new_parts.get(0).isSetCatName()) {
- final String defaultCat = getDefaultCatalog(conf);
- new_parts.forEach(p -> p.setCatName(defaultCat));
+ if (hiveVersion == HiveVersion.V3_0) {
+ final String defaultCat = getDefaultCatalog(conf);
+ new_parts.forEach(p -> p.setCatName(defaultCat));
+ }
}
return client.add_partitions(new_parts);
}
@@ -931,24 +944,24 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
@Override
public Partition appendPartition(String catName, String dbName, String tableName,
String name) throws TException {
- Partition p = client.append_partition_by_name(prependCatalogToDbName(
- catName, dbName, conf), tableName, name);
+ Partition p = client.append_partition_by_name(
+ prependCatalogToDbNameByVersion(hiveVersion, catName, dbName, conf), tableName, name);
return deepCopy(p);
}
@Override
public Partition appendPartition(String catName, String dbName, String tableName,
List<String> partVals) throws TException {
- Partition p = client.append_partition(prependCatalogToDbName(
- catName, dbName, conf), tableName, partVals);
+ Partition p = client.append_partition(
+ prependCatalogToDbNameByVersion(hiveVersion, catName, dbName, conf), tableName, partVals);
return deepCopy(p);
}
@Deprecated
public Partition appendPartition(String dbName, String tableName, List<String> partVals,
EnvironmentContext ec) throws TException {
- return client.append_partition_with_environment_context(prependCatalogToDbName(dbName, conf),
- tableName, partVals, ec).deepCopy();
+ return client.append_partition_with_environment_context(
+ prependCatalogToDbNameByVersion(hiveVersion, null, dbName, conf), tableName, partVals, ec).deepCopy();
}
/**
@@ -1260,7 +1273,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
}
}
}
- client.drop_database(prependCatalogToDbName(catalogName, dbName, conf), deleteData, cascade);
+ client.drop_database(prependCatalogToDbNameByVersion(hiveVersion, catalogName, dbName, conf), deleteData, cascade);
}
@Override
@@ -1272,7 +1285,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
@Override
public boolean dropPartition(String catName, String db_name, String tbl_name, String name,
boolean deleteData) throws TException {
- return client.drop_partition_by_name_with_environment_context(prependCatalogToDbName(
+ return client.drop_partition_by_name_with_environment_context(prependCatalogToDbNameByVersion(hiveVersion,
catName, db_name, conf), tbl_name, name, deleteData, null);
}
@@ -1289,21 +1302,23 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
@Deprecated
public boolean dropPartition(String db_name, String tbl_name, List<String> part_vals,
EnvironmentContext env_context) throws TException {
- return client.drop_partition_with_environment_context(prependCatalogToDbName(db_name, conf),
- tbl_name, part_vals, true, env_context);
+ return client.drop_partition_with_environment_context(
+ prependCatalogToDbNameByVersion(hiveVersion, null, db_name, conf), tbl_name, part_vals, true, env_context);
}
@Deprecated
public boolean dropPartition(String dbName, String tableName, String partName, boolean dropData,
EnvironmentContext ec) throws TException {
- return client.drop_partition_by_name_with_environment_context(prependCatalogToDbName(dbName, conf),
+ return client.drop_partition_by_name_with_environment_context(
+ prependCatalogToDbNameByVersion(hiveVersion, null, dbName, conf),
tableName, partName, dropData, ec);
}
@Deprecated
public boolean dropPartition(String dbName, String tableName, List<String> partVals)
throws TException {
- return client.drop_partition(prependCatalogToDbName(dbName, conf), tableName, partVals, true);
+ return client.drop_partition(prependCatalogToDbNameByVersion(hiveVersion, null, dbName, conf),
+ tableName, partVals, true);
}
@Override
@@ -1340,7 +1355,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
}
}
}
- return client.drop_partition_with_environment_context(prependCatalogToDbName(
+ return client.drop_partition_with_environment_context(prependCatalogToDbNameByVersion(hiveVersion,
catName, db_name, conf), tbl_name, part_vals, options.deleteData,
options.purgeData ? getEnvironmentContextWithIfPurgeSet() : null);
}
@@ -1391,7 +1406,9 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
}
rps.setExprs(exprs);
DropPartitionsRequest req = new DropPartitionsRequest(dbName, tblName, rps);
- req.setCatName(catName);
+ if (hiveVersion == HiveVersion.V3_0) {
+ req.setCatName(catName);
+ }
req.setDeleteData(options.deleteData);
req.setNeedResult(options.returnResults);
req.setIfExists(options.ifExists);
@@ -1501,7 +1518,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
@Override
public void truncateTable(String catName, String dbName, String tableName, List<String> partNames)
throws TException {
- client.truncate_table(prependCatalogToDbName(catName, dbName, conf), tableName, partNames);
+ client.truncate_table(prependCatalogToDbNameByVersion(hiveVersion, catName, dbName, conf), tableName, partNames);
}
/**
@@ -1581,7 +1598,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
@Override
public List<Partition> listPartitions(String catName, String db_name, String tbl_name,
int max_parts) throws TException {
- List<Partition> parts = client.get_partitions(prependCatalogToDbName(catName, db_name, conf),
+ List<Partition> parts = client.get_partitions(prependCatalogToDbNameByVersion(hiveVersion, catName, db_name, conf),
tbl_name, shrinkMaxtoShort(max_parts));
return deepCopyPartitions(filterHook.filterPartitions(parts));
}
@@ -1607,7 +1624,8 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
@Override
public List<Partition> listPartitions(String catName, String db_name, String tbl_name,
List<String> part_vals, int max_parts) throws TException {
- List<Partition> parts = client.get_partitions_ps(prependCatalogToDbName(catName, db_name, conf),
+ List<Partition> parts = client.get_partitions_ps(
+ prependCatalogToDbNameByVersion(hiveVersion, catName, db_name, conf),
tbl_name, part_vals, shrinkMaxtoShort(max_parts));
return deepCopyPartitions(filterHook.filterPartitions(parts));
}
@@ -1624,7 +1642,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
public List<Partition> listPartitionsWithAuthInfo(String catName, String dbName, String tableName,
int maxParts, String userName,
List<String> groupNames) throws TException {
- List<Partition> parts = client.get_partitions_with_auth(prependCatalogToDbName(catName,
+ List<Partition> parts = client.get_partitions_with_auth(prependCatalogToDbNameByVersion(hiveVersion, catName,
dbName, conf), tableName, shrinkMaxtoShort(maxParts), userName, groupNames);
return deepCopyPartitions(filterHook.filterPartitions(parts));
}
@@ -1643,8 +1661,8 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
List<String> partialPvals, int maxParts,
String userName, List<String> groupNames)
throws TException {
- List<Partition> parts = client.get_partitions_ps_with_auth(prependCatalogToDbName(catName,
- dbName, conf), tableName, partialPvals, shrinkMaxtoShort(maxParts), userName, groupNames);
+ List<Partition> parts = client.get_partitions_ps_with_auth(prependCatalogToDbNameByVersion(hiveVersion,
+ catName, dbName, conf), tableName, partialPvals, shrinkMaxtoShort(maxParts), userName, groupNames);
return deepCopyPartitions(filterHook.filterPartitions(parts));
}
@@ -1784,8 +1802,8 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
public Partition getPartitionWithAuthInfo(String catName, String dbName, String tableName,
List<String> pvals, String userName,
List<String> groupNames) throws TException {
- Partition p = client.get_partition_with_auth(prependCatalogToDbName(catName, dbName, conf), tableName,
- pvals, userName, groupNames);
+ Partition p = client.get_partition_with_auth(prependCatalogToDbNameByVersion(hiveVersion, catName, dbName, conf),
+ tableName, pvals, userName, groupNames);
return deepCopy(filterHook.filterPartition(p));
}
@@ -1866,7 +1884,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
public List<String> listTableNamesByFilter(String catName, String dbName, String filter,
int maxTables) throws TException {
return filterHook.filterTableNames(catName, dbName,
- client.get_table_names_by_filter(prependCatalogToDbName(catName, dbName, conf), filter,
+ client.get_table_names_by_filter(prependCatalogToDbNameByVersion(hiveVersion, catName, dbName, conf), filter,
shrinkMaxtoShort(maxTables)));
}
@@ -2065,8 +2083,8 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
@Override
public void alter_partition(String catName, String dbName, String tblName, Partition newPart,
EnvironmentContext environmentContext) throws TException {
- client.alter_partition_with_environment_context(prependCatalogToDbName(catName, dbName, conf), tblName,
- newPart, environmentContext);
+ client.alter_partition_with_environment_context(prependCatalogToDbNameByVersion(hiveVersion, catName, dbName, conf),
+ tblName, newPart, environmentContext);
}
@Override
@@ -2085,7 +2103,8 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
public void alter_partitions(String catName, String dbName, String tblName,
List<Partition> newParts,
EnvironmentContext environmentContext) throws TException {
- client.alter_partitions_with_environment_context(prependCatalogToDbName(catName, dbName, conf),
+ client.alter_partitions_with_environment_context(
+ prependCatalogToDbNameByVersion(hiveVersion, catName, dbName, conf),
tblName, newParts, environmentContext);
}
@@ -2096,7 +2115,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
@Override
public void alterDatabase(String catName, String dbName, Database newDb) throws TException {
- client.alter_database(prependCatalogToDbName(catName, dbName, conf), newDb);
+ client.alter_database(prependCatalogToDbNameByVersion(hiveVersion, catName, dbName, conf), newDb);
}
@Override
@@ -2249,7 +2268,8 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
public boolean deletePartitionColumnStatistics(String catName, String dbName, String tableName,
String partName, String colName)
throws TException {
- return client.delete_partition_column_statistics(prependCatalogToDbName(catName, dbName, conf),
+ return client.delete_partition_column_statistics(
+ prependCatalogToDbNameByVersion(hiveVersion, catName, dbName, conf),
tableName, partName, colName);
}
@@ -2262,7 +2282,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
@Override
public boolean deleteTableColumnStatistics(String catName, String dbName, String tableName,
String colName) throws TException {
- return client.delete_table_column_statistics(prependCatalogToDbName(catName, dbName, conf),
+ return client.delete_table_column_statistics(prependCatalogToDbNameByVersion(hiveVersion, catName, dbName, conf),
tableName, colName);
}
@@ -2304,8 +2324,8 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
@Override
public Partition getPartition(String catName, String dbName, String tblName, String name)
throws TException {
- Partition p = client.get_partition_by_name(prependCatalogToDbName(catName, dbName, conf), tblName,
- name);
+ Partition p = client.get_partition_by_name(prependCatalogToDbNameByVersion(hiveVersion, catName, dbName, conf),
+ tblName, name);
return deepCopy(filterHook.filterPartition(p));
}
@@ -3144,7 +3164,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
protected void drop_table_with_environment_context(String catName, String dbname, String name,
boolean deleteData, EnvironmentContext envContext) throws TException {
- client.drop_table_with_environment_context(prependCatalogToDbName(catName, dbname, conf),
+ client.drop_table_with_environment_context(prependCatalogToDbNameByVersion(hiveVersion, catName, dbname, conf),
name, deleteData, envContext);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 14/15: [fix] (Nereids) fix date function rewrite on datetimev1 column bug (#32569)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit b9788e5e37a79e794b9ff7a95b57c7bf193275e3
Author: Ziyan Lee <49...@users.noreply.github.com>
AuthorDate: Sat Mar 23 22:15:39 2024 +0800
[fix] (Nereids) fix date function rewrite on datetimev1 column bug (#32569)
---
.../trees/expressions/literal/DateLiteral.java | 4 +-
.../datetimev1/test_datetimev1_date_function.out | 32 ++++++
.../test_datetimev1_date_function.groovy | 124 +++++++++++++++++++++
3 files changed, 158 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java
index 7dcd86a1504..a33cc32c16f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java
@@ -458,10 +458,10 @@ public class DateLiteral extends Literal {
/**
* 2020-01-01
*
- * @return 2020-01-01 24:00:00
+ * @return 2020-01-01 23:59:59
*/
public DateTimeLiteral toEndOfTheDay() {
- return new DateTimeLiteral(year, month, day, 24, 0, 0);
+ return new DateTimeLiteral(year, month, day, 23, 59, 59);
}
/**
diff --git a/regression-test/data/datatype_p0/datetimev1/test_datetimev1_date_function.out b/regression-test/data/datatype_p0/datetimev1/test_datetimev1_date_function.out
new file mode 100644
index 00000000000..81a89b2286f
--- /dev/null
+++ b/regression-test/data/datatype_p0/datetimev1/test_datetimev1_date_function.out
@@ -0,0 +1,32 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !date_function_test_datetimev1_gt --
+2020-01-06T21:02:03
+2020-01-07T09:29:39
+2020-01-08T11:33:55
+2020-01-09T14:43:42
+2020-01-10T23:59:59
+
+-- !date_function_test_datetimev1_gte --
+2020-01-05T17:58:59
+2020-01-06T21:02:03
+2020-01-07T09:29:39
+2020-01-08T11:33:55
+2020-01-09T14:43:42
+2020-01-10T23:59:59
+
+-- !date_function_test_datetimev1_lt --
+2020-01-01T19:29:39
+2020-01-02T18:28:38
+2020-01-03T23:37:47
+2020-01-04T21:42:43
+
+-- !date_function_test_datetimev1_lte --
+2020-01-01T19:29:39
+2020-01-02T18:28:38
+2020-01-03T23:37:47
+2020-01-04T21:42:43
+2020-01-05T17:58:59
+
+-- !date_function_test_datetimev1_eq --
+2020-01-05T17:58:59
+
diff --git a/regression-test/suites/datatype_p0/datetimev1/test_datetimev1_date_function.groovy b/regression-test/suites/datatype_p0/datetimev1/test_datetimev1_date_function.groovy
new file mode 100644
index 00000000000..ec473a82027
--- /dev/null
+++ b/regression-test/suites/datatype_p0/datetimev1/test_datetimev1_date_function.groovy
@@ -0,0 +1,124 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_datetimev1_date_function", "nonConcurrent") {
+
+ /// legacy date/datetime format need to run in non concurrent mode.
+ sql """
+ admin set frontend config("enable_date_conversion" = "false");
+ """
+
+ def table_dup = "test_datetimev1_date_function_dup_tbl" // duplicate key
+
+ sql "drop table if exists ${table_dup};"
+
+ sql """
+ CREATE TABLE IF NOT EXISTS `${table_dup}` (
+ `date_key1` datetimev1 NULL COMMENT "",
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`date_key1`)
+ COMMENT "OLAP"
+ DISTRIBUTED BY HASH(`date_key1`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "in_memory" = "false",
+ "storage_format" = "V2"
+ )
+ """
+
+ def insert_data = { table_name ->
+ sql """
+ insert into ${table_name}
+ values
+ ('2020-01-01 19:29:39'),
+ ('2020-01-02 18:28:38'),
+ ('2020-01-03 23:37:47'),
+ ('2020-01-04 21:42:43'),
+ ('2020-01-05 17:58:59'),
+ ('2020-01-06 21:02:03'),
+ ('2020-01-07 09:29:39'),
+ ('2020-01-08 11:33:55'),
+ ('2020-01-09 14:43:42'),
+ ('2020-01-10 23:59:59');
+ """
+ }
+
+ insert_data(table_dup)
+
+ def run_compare_test = { table_name, col, col_value ->
+ def query1 = """
+ select
+ *
+ from
+ `${table_name}`
+ where
+ date(`${col}`) > ${col_value}
+ order by 1;
+ """
+ quickTest("date_function_test_datetimev1_gt", query1)
+
+ def query2 = """
+ select
+ *
+ from
+ `${table_name}`
+ where
+ date(`${col}`) >= ${col_value}
+ order by 1;
+ """
+ quickTest("date_function_test_datetimev1_gte", query2)
+
+ def query3 = """
+ select
+ *
+ from
+ `${table_name}`
+ where
+ date(`${col}`) < ${col_value}
+ order by 1;
+ """
+ quickTest("date_function_test_datetimev1_lt", query3)
+
+ def query4 = """
+ select
+ *
+ from
+ `${table_name}`
+ where
+ date(`${col}`) <= ${col_value}
+ order by 1;
+ """
+ quickTest("date_function_test_datetimev1_lte", query4)
+
+ def query5 = """
+ select
+ *
+ from
+ `${table_name}`
+ where
+ date(`${col}`) = ${col_value}
+ order by 1;
+ """
+ quickTest("date_function_test_datetimev1_eq", query5)
+ }
+
+ run_compare_test(table_dup, "date_key1", "'2020-01-05'")
+
+ sql """
+ admin set frontend config("enable_date_conversion" = "true");
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org