You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/01/18 02:04:52 UTC
(doris) branch branch-2.1 updated (b758286b575 -> e7c6c12a65a)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a change to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
from b758286b575 [doc](Variant) add doc for Variant type (#28951)
new 3deee146805 [fix](Nereids): find hash condition after infer predicate (#30026)
new 6ef9ed08aad [fix](multi-table-load) fix multi table load can not finish (#29957)
new 2ca3c821351 [fix](fe ut) fix unstable TabletRepairAndBalanceTest (#30041)
new b0c49024cb7 [Feature](materialized-view) support match function with alias in materialized-view (#30025)
new d4195b3cc01 [fix](Nereids): fix enable mv rewrite is useless in dphyp (#30042)
new e73c0f40779 [fix](fe) Fix drop table write too many times editlog (#29966)
new 7e3ed8a19a6 [fix](index_tool) fix index_tool compile error (#30049)
new 48d7c1b1edd [test](regression-test) fix case, compatible with 3 replicas. (#29905)
new dcc9cf9dd3d [config](Nereids): add SessionVariable to control nereids timeout (#30048)
new e894911cdaa [function](char) change char function behaviour same with mysql (#30034)
new 143df9f414b [typo](docs) fix typo (#30031)
new 94cbd69ccae [Fix](compile) fix compile failure on newest clang (#30056)
new d1acd42db4d [Improve](config)delete confused config for nested complex type (#29988)
new 40d5e664d6d [feature](statistics)support statistics for iceberg/paimon/hudi table (#29868)
new 29619819351 [Chore](config) remove some unused config (#29983)
new 6c9ec7b479c [minor](Nereids): remove useless method isMysqlCompatibleDatabase (#30064)
new d0977824e93 [cleanup](insert-into) clean up some insert into log (#30063)
new 5fcdf63a394 [Fix](typo) Fix group commit regression test typo (#30057)
new 77230172537 [Performance](Join) Support all match one logic (#30019)
new cbac86324ad [feature](statistics, metadata)Meta data place holder for statistics (#29867)
new cb5fe27b5ac [fix](ut) fixed test cases failure for showCreateFunction (#28593)
new e7c6c12a65a [improvement](fe) skip attach profile during send fragment (#30085)
The 22 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/common/config.cpp | 8 -
be/src/common/config.h | 8 -
be/src/index-tools/index_tool.cpp | 15 +-
be/src/io/fs/multi_table_pipe.cpp | 32 +--
be/src/io/fs/multi_table_pipe.h | 19 +-
be/src/pipeline/exec/union_source_operator.cpp | 6 -
be/src/runtime/fragment_mgr.cpp | 5 -
.../routine_load/routine_load_task_executor.cpp | 2 +-
be/src/service/backend_service.cpp | 4 +-
be/src/vec/exec/join/process_hash_table_probe.h | 4 +-
.../vec/exec/join/process_hash_table_probe_impl.h | 24 +-
be/src/vec/exec/vunion_node.cpp | 6 -
be/src/vec/functions/function_string.h | 6 +-
.../docker-compose/iceberg/iceberg.env | 2 +-
.../docker-compose/iceberg/iceberg.yaml.tpl | 2 +
docs/en/docs/admin-manual/config/be-config.md | 18 --
docs/zh-CN/docs/admin-manual/config/be-config.md | 18 --
docs/zh-CN/docs/advanced/variables.md | 2 +-
.../main/java/org/apache/doris/common/Config.java | 11 -
.../org/apache/doris/common/FeMetaVersion.java | 4 +-
.../java/org/apache/doris/analysis/Analyzer.java | 2 +
.../java/org/apache/doris/analysis/DropDbStmt.java | 4 +-
.../org/apache/doris/analysis/FunctionName.java | 4 +
.../java/org/apache/doris/analysis/TypeDef.java | 11 -
.../java/org/apache/doris/catalog/Database.java | 13 +-
.../java/org/apache/doris/catalog/DatabaseIf.java | 2 +-
.../doris/catalog/MysqlCompatibleDatabase.java | 5 -
.../main/java/org/apache/doris/catalog/Tablet.java | 4 +
.../catalog/external/IcebergExternalTable.java | 9 +
.../catalog/external/PaimonExternalTable.java | 9 +
.../java/org/apache/doris/clone/TabletChecker.java | 3 +-
.../apache/doris/common/proc/CatalogsProcDir.java | 10 +-
.../org/apache/doris/datasource/CatalogIf.java | 6 +-
.../apache/doris/datasource/InternalCatalog.java | 19 +-
.../org/apache/doris/httpv2/rest/ShowAction.java | 5 +-
.../master/PartitionInMemoryInfoCollector.java | 3 +-
.../jobs/cascades/OptimizeGroupExpressionJob.java | 23 +-
.../doris/nereids/jobs/executor/Rewriter.java | 3 +-
.../nereids/jobs/scheduler/SimpleJobScheduler.java | 12 +-
.../org/apache/doris/nereids/rules/RuleSet.java | 1 -
.../plans/commands/info/ColumnDefinition.java | 14 --
.../java/org/apache/doris/persist/EditLog.java | 26 ++-
.../org/apache/doris/persist/OperationType.java | 6 +
.../main/java/org/apache/doris/qe/Coordinator.java | 27 +--
.../java/org/apache/doris/qe/SessionVariable.java | 3 +
.../org/apache/doris/rewrite/FunctionAlias.java | 59 +++++
.../apache/doris/statistics/AnalysisJobInfo.java | 54 +++++
.../apache/doris/statistics/AnalysisManager.java | 60 +----
.../apache/doris/statistics/AnalysisTaskInfo.java | 58 +++++
...AnalysisTask.java => ExternalAnalysisTask.java} | 173 ++------------
.../apache/doris/statistics/HMSAnalysisTask.java | 252 ++-------------------
.../doris/statistics/NewPartitionLoadedEvent.java | 57 +++++
.../apache/doris/statistics/UpdateRowsEvent.java | 61 +++++
.../apache/doris/statistics/util/SimpleQueue.java | 65 ------
.../doris/statistics/util/StatisticsUtil.java | 10 +-
.../transaction/DbUsedDataQuotaInfoCollector.java | 3 +-
.../doris/analysis/ShowCreateFunctionTest.java | 12 +-
.../org/apache/doris/clone/RebalancerTestUtil.java | 3 +-
.../doris/clone/TabletRepairAndBalanceTest.java | 4 +-
.../doris/statistics/AnalysisManagerTest.java | 24 --
.../doris/statistics/HMSAnalysisTaskTest.java | 4 +-
gensrc/script/doris_builtins_functions.py | 2 +-
regression-test/conf/regression-conf.groovy | 1 +
.../iceberg/test_iceberg_statistics.out | 39 ++++
.../paimon/test_paimon_statistics.out | 21 ++
...ocal_tvf_with_complex_type_insertinto_doris.out | 3 -
.../hive/test_hive_hudi_statistics.out | 16 ++
.../data/mv_p0/test_substr/test_substr.out | 4 +
.../data/nereids_p0/hint/fix_leading.out | 4 +-
.../data/nereids_p0/hint/test_leading.out | 16 +-
.../string_functions/test_string_function.out | Bin 3360 -> 3418 bytes
.../org/apache/doris/regression/suite/Suite.groovy | 27 ++-
regression-test/pipeline/p0/conf/be.conf | 2 +-
.../pipeline/p0/conf/regression-conf.groovy | 1 +
.../tpch/tpch-sf100/conf/regression-conf.groovy | 1 +
.../plugins/plugins_get_ids_from_proc.groovy | 62 +++++
.../three_level_nestedtypes_with_s3data.groovy | 2 +-
.../two_level_nestedtypes_with_s3data.groovy | 1 -
.../suites/datatype_p0/nested_types/load.groovy | 2 +-
.../map_functions/test_basic_map_function.groovy | 1 -
.../query/test_nested_type_with_count.groovy | 1 -
...st_nested_types_insert_into_with_literal.groovy | 1 -
.../test_nested_types_insert_into_with_s3.groovy | 1 -
...test_nestedtypes_csv_insert_into_with_s3.groovy | 1 -
.../test_nestedtypes_insert_into_select.groovy | 1 -
...est_nestedtypes_json_insert_into_with_s3.groovy | 1 -
.../iceberg/test_iceberg_statistics.groovy | 57 +++++
.../paimon/test_paimon_catalog.groovy | 3 +
.../paimon/test_paimon_statistics.groovy | 47 ++++
.../external_table_p0/test_catalog_ddl.groovy | 2 +
...l_tvf_with_complex_type_insertinto_doris.groovy | 2 +-
.../hive/test_hive_hudi_statistics.groovy | 47 ++++
.../test_low_wal_disk_space_fault_injection.groovy | 2 +-
.../test_index_change_with_compaction.groovy | 13 +-
.../test_index_compaction_dup_keys.groovy | 10 +-
.../test_index_compaction_unique_keys.groovy | 10 +-
...dex_compaction_with_multi_index_segments.groovy | 10 +-
.../stream_load/test_load_to_single_tablet.groovy | 4 +-
.../test_load_with_map_nested_array.groovy | 1 -
.../suites/mv_p0/test_substr/test_substr.groovy | 56 +++++
.../select_tablets/select_with_tablets.groovy | 3 +
.../aggregate/map_agg_nested_insert_doris.groovy | 2 +-
.../show/test_nested_complex_switch.groovy | 56 -----
.../string_functions/test_string_function.groovy | 2 +
.../suites/update/test_update_configs.groovy | 15 +-
105 files changed, 1010 insertions(+), 862 deletions(-)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/rewrite/FunctionAlias.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJobInfo.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskInfo.java
copy fe/fe-core/src/main/java/org/apache/doris/statistics/{HMSAnalysisTask.java => ExternalAnalysisTask.java} (59%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/statistics/NewPartitionLoadedEvent.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/statistics/UpdateRowsEvent.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/statistics/util/SimpleQueue.java
create mode 100644 regression-test/data/external_table_p0/iceberg/test_iceberg_statistics.out
create mode 100644 regression-test/data/external_table_p0/paimon/test_paimon_statistics.out
create mode 100644 regression-test/data/external_table_p2/hive/test_hive_hudi_statistics.out
create mode 100644 regression-test/data/mv_p0/test_substr/test_substr.out
create mode 100644 regression-test/plugins/plugins_get_ids_from_proc.groovy
create mode 100644 regression-test/suites/external_table_p0/iceberg/test_iceberg_statistics.groovy
create mode 100644 regression-test/suites/external_table_p0/paimon/test_paimon_statistics.groovy
create mode 100644 regression-test/suites/external_table_p2/hive/test_hive_hudi_statistics.groovy
create mode 100644 regression-test/suites/mv_p0/test_substr/test_substr.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 07/22: [fix](index_tool) fix index_tool compile error (#30049)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 7e3ed8a19a682d1448b2a8fd84e60a4ddb4e39b6
Author: zzzxl <33...@users.noreply.github.com>
AuthorDate: Wed Jan 17 17:04:44 2024 +0800
[fix](index_tool) fix index_tool compile error (#30049)
---
be/src/index-tools/index_tool.cpp | 15 +++++++++++----
1 file changed, 11 insertions(+), 4 deletions(-)
diff --git a/be/src/index-tools/index_tool.cpp b/be/src/index-tools/index_tool.cpp
index 1ad624d12da..ade72ae6809 100644
--- a/be/src/index-tools/index_tool.cpp
+++ b/be/src/index-tools/index_tool.cpp
@@ -17,11 +17,13 @@
#include <CLucene.h>
#include <CLucene/config/repl_wchar.h>
+#include <gen_cpp/PaloInternalService_types.h>
#include <gflags/gflags.h>
#include <cstdint>
#include <filesystem>
#include <iostream>
+#include <memory>
#include <roaring/roaring.hh>
#include <sstream>
#include <string>
@@ -29,12 +31,14 @@
#include "io/fs/local_file_system.h"
#include "olap/rowset/segment_v2/inverted_index/query/conjunction_query.h"
+#include "olap/rowset/segment_v2/inverted_index/query/query.h"
#include "olap/rowset/segment_v2/inverted_index_compound_directory.h"
#include "olap/rowset/segment_v2/inverted_index_compound_reader.h"
using doris::segment_v2::DorisCompoundReader;
using doris::segment_v2::DorisCompoundDirectoryFactory;
using doris::io::FileInfo;
+using namespace doris::segment_v2;
using namespace lucene::analysis;
using namespace lucene::index;
using namespace lucene::util;
@@ -83,7 +87,7 @@ void search(lucene::store::Directory* dir, std::string& field, std::string& toke
_CLDELETE(reader);
reader = newreader;
}
- IndexSearcher s(reader);
+ auto s = std::make_shared<IndexSearcher>(reader);
std::unique_ptr<lucene::search::Query> query;
std::cout << "version: " << (int32_t)(reader->getIndexVersion()) << std::endl;
@@ -124,13 +128,16 @@ void search(lucene::store::Directory* dir, std::string& field, std::string& toke
if (pred == "match_all") {
roaring::Roaring result;
std::vector<std::string> terms = split(token, '|');
- doris::ConjunctionQuery query(s.getReader());
+
+ doris::TQueryOptions queryOptions;
+ ConjunctionQuery query(s, queryOptions);
query.add(field_ws, terms);
query.search(result);
+
total += result.cardinality();
} else {
roaring::Roaring result;
- s._search(query.get(), [&result](const int32_t docid, const float_t /*score*/) {
+ s->_search(query.get(), [&result](const int32_t docid, const float_t /*score*/) {
// docid equal to rowid in segment
result.add(docid);
if (FLAGS_print_row_id) {
@@ -141,7 +148,7 @@ void search(lucene::store::Directory* dir, std::string& field, std::string& toke
}
std::cout << "Term queried count:" << total << std::endl;
- s.close();
+ s->close();
reader->close();
_CLLDELETE(reader);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 01/22: [fix](Nereids): find hash condition after infer predicate (#30026)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 3deee146805b66d5548842f28e9f64ae317bb6c6
Author: jakevin <ja...@gmail.com>
AuthorDate: Wed Jan 17 11:08:18 2024 +0800
[fix](Nereids): find hash condition after infer predicate (#30026)
---
.../org/apache/doris/nereids/jobs/executor/Rewriter.java | 3 ++-
regression-test/data/nereids_p0/hint/fix_leading.out | 4 ++--
regression-test/data/nereids_p0/hint/test_leading.out | 16 ++++++++--------
3 files changed, 12 insertions(+), 11 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index cbfe3d28ac8..b2f1569ea1d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -226,7 +226,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
// but top-down traverse can not cover this case in one iteration, so bottom-up is more
// efficient because it can find the new plans and apply transform wherever it is
bottomUp(RuleSet.PUSH_DOWN_FILTERS),
- //after push down, some new filters are generated, which needs to be optimized. (example: tpch q19)
+ // after push down, some new filters are generated, which needs to be optimized. (example: tpch q19)
topDown(new ExpressionOptimization()),
topDown(
new MergeFilters(),
@@ -263,6 +263,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
// after eliminate outer join, we can move some filters to join.otherJoinConjuncts,
// this can help to translate plan to backend
topDown(new PushFilterInsideJoin()),
+ topDown(new FindHashConditionForJoin()),
topDown(new ExpressionNormalization())
),
diff --git a/regression-test/data/nereids_p0/hint/fix_leading.out b/regression-test/data/nereids_p0/hint/fix_leading.out
index 898fe5882b9..1410c8e0e6e 100644
--- a/regression-test/data/nereids_p0/hint/fix_leading.out
+++ b/regression-test/data/nereids_p0/hint/fix_leading.out
@@ -9,9 +9,9 @@ PhysicalResultSink
----------PhysicalDistribute[DistributionSpecHash]
------------PhysicalOlapScan[t2]
--------PhysicalDistribute[DistributionSpecHash]
-----------NestedLoopJoin[CROSS_JOIN](t3.c3 = t4.c4)
+----------hashJoin[INNER_JOIN] hashCondition=((t3.c3 = t4.c4)) otherCondition=()
------------PhysicalOlapScan[t3]
-------------PhysicalDistribute[DistributionSpecReplicated]
+------------PhysicalDistribute[DistributionSpecHash]
--------------PhysicalOlapScan[t4]
Hint log:
diff --git a/regression-test/data/nereids_p0/hint/test_leading.out b/regression-test/data/nereids_p0/hint/test_leading.out
index fe3831a9fc4..29811a95161 100644
--- a/regression-test/data/nereids_p0/hint/test_leading.out
+++ b/regression-test/data/nereids_p0/hint/test_leading.out
@@ -2609,7 +2609,7 @@ PhysicalResultSink
------------PhysicalProject
--------------PhysicalOlapScan[t2]
------------PhysicalDistribute[DistributionSpecReplicated]
---------------NestedLoopJoin[CROSS_JOIN](t1.c1 = t3.c3)
+--------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t3.c3)) otherCondition=()
----------------PhysicalProject
------------------PhysicalOlapScan[t1]
----------------PhysicalDistribute[DistributionSpecReplicated]
@@ -2631,7 +2631,7 @@ PhysicalResultSink
------------PhysicalProject
--------------PhysicalOlapScan[t2]
------------PhysicalDistribute[DistributionSpecReplicated]
---------------NestedLoopJoin[CROSS_JOIN](t1.c1 = t3.c3)
+--------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t3.c3)) otherCondition=()
----------------PhysicalProject
------------------PhysicalOlapScan[t3]
----------------PhysicalDistribute[DistributionSpecReplicated]
@@ -2745,7 +2745,7 @@ PhysicalResultSink
------------PhysicalProject
--------------PhysicalOlapScan[t2]
------------PhysicalDistribute[DistributionSpecReplicated]
---------------NestedLoopJoin[CROSS_JOIN](t1.c1 = t3.c3)
+--------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t3.c3)) otherCondition=()
----------------PhysicalProject
------------------PhysicalOlapScan[t1]
----------------PhysicalDistribute[DistributionSpecReplicated]
@@ -2767,7 +2767,7 @@ PhysicalResultSink
------------PhysicalProject
--------------PhysicalOlapScan[t2]
------------PhysicalDistribute[DistributionSpecReplicated]
---------------NestedLoopJoin[CROSS_JOIN](t1.c1 = t3.c3)
+--------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t3.c3)) otherCondition=()
----------------PhysicalProject
------------------PhysicalOlapScan[t3]
----------------PhysicalDistribute[DistributionSpecReplicated]
@@ -2881,10 +2881,10 @@ PhysicalResultSink
------------PhysicalProject
--------------PhysicalOlapScan[t2]
------------PhysicalDistribute[DistributionSpecHash]
---------------NestedLoopJoin[CROSS_JOIN](t1.c1 = t3.c3)
+--------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t3.c3)) otherCondition=()
----------------PhysicalProject
------------------PhysicalOlapScan[t1]
-----------------PhysicalDistribute[DistributionSpecReplicated]
+----------------PhysicalDistribute[DistributionSpecHash]
------------------PhysicalProject
--------------------PhysicalOlapScan[t3]
@@ -2903,10 +2903,10 @@ PhysicalResultSink
------------PhysicalProject
--------------PhysicalOlapScan[t2]
------------PhysicalDistribute[DistributionSpecHash]
---------------NestedLoopJoin[CROSS_JOIN](t1.c1 = t3.c3)
+--------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t3.c3)) otherCondition=()
----------------PhysicalProject
------------------PhysicalOlapScan[t3]
-----------------PhysicalDistribute[DistributionSpecReplicated]
+----------------PhysicalDistribute[DistributionSpecHash]
------------------PhysicalProject
--------------------PhysicalOlapScan[t1]
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 11/22: [typo](docs) fix typo (#30031)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 143df9f414bfa36b514a0793fe5bf8821d4a2d8d
Author: hechao <73...@users.noreply.github.com>
AuthorDate: Wed Jan 17 17:16:12 2024 +0800
[typo](docs) fix typo (#30031)
Co-authored-by: hechao <he...@selectdb.com>
---
docs/zh-CN/docs/advanced/variables.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/zh-CN/docs/advanced/variables.md b/docs/zh-CN/docs/advanced/variables.md
index 9c042d5049f..3075c46a1c2 100644
--- a/docs/zh-CN/docs/advanced/variables.md
+++ b/docs/zh-CN/docs/advanced/variables.md
@@ -565,7 +565,7 @@ try (Connection conn = DriverManager.getConnection("jdbc:mysql://127.0.0.1:9030/
* `skip_missing_version`
- 有些极端场景下，表的 Tablet 下的所有的所有副本都有版本缺失，使得这些 Tablet 没有办法被恢复，导致整张表都不能查询。这个变量可以用来控制查询的行为，打设置为`true`时，查询会忽略 FE partition 中记录的 visibleVersion，使用 replica version。如果 Be 上的 Replica 有缺失的版本，则查询会直接跳过这些缺失的版本，只返回仍存在版本的数据。此外，查询将会总是选择所有存活的 BE 中所有 Replica 里 lastSuccessVersion 最大的那一个，这样可以尽可能的恢复更多的数据。这个变量应该只在上述紧急情况下才被设置为`true`，仅用于临时让表恢复查询。注意，此变量与 use_fix_replica 变量冲突，当 use_fix_replica 变量不等于 -1 时，此变量会不起作用
+ 有些极端场景下，表的 Tablet 下的所有的所有副本都有版本缺失，使得这些 Tablet 没有办法被恢复，导致整张表都不能查询。这个变量可以用来控制查询的行为，当设置为`true`时，查询会忽略 FE partition 中记录的 visibleVersion，使用 replica version。如果 Be 上的 Replica 有缺失的版本，则查询会直接跳过这些缺失的版本，只返回仍存在版本的数据。此外，查询将会总是选择所有存活的 BE 中所有 Replica 里 lastSuccessVersion 最大的那一个，这样可以尽可能的恢复更多的数据。这个变量应该只在上述紧急情况下才被设置为`true`，仅用于临时让表恢复查询。注意，此变量与 use_fix_replica 变量冲突，当 use_fix_replica 变量不等于 -1 时，此变量会不起作用
* `default_password_lifetime`
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 15/22: [Chore](config) remove some unused config (#29983)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 296198193519ecd3df7345d85cfbc4878289a15f
Author: Pxl <px...@qq.com>
AuthorDate: Wed Jan 17 17:49:59 2024 +0800
[Chore](config) remove some unused config (#29983)
remove some unused config
---
be/src/common/config.cpp | 8 --------
be/src/common/config.h | 8 --------
docs/en/docs/admin-manual/config/be-config.md | 18 ------------------
docs/zh-CN/docs/admin-manual/config/be-config.md | 18 ------------------
.../suites/update/test_update_configs.groovy | 15 ++-------------
5 files changed, 2 insertions(+), 65 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index b678d855594..d919d7f5719 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -265,8 +265,6 @@ DEFINE_mInt32(doris_max_scan_key_num, "48");
// the max number of push down values of a single column.
// if exceed, no conditions will be pushed down for that column.
DEFINE_mInt32(max_pushdown_conditions_per_column, "1024");
-// return_row / total_row
-DEFINE_mInt32(doris_max_pushdown_conjuncts_return_rate, "90");
// (Advanced) Maximum size of per-query receive-side buffer
DEFINE_mInt32(exchg_node_buffer_size_bytes, "20485760");
@@ -542,10 +540,6 @@ DEFINE_Int32(min_buffer_size, "1024"); // 1024, The minimum read buffer size (in
// With 1024B through 8MB buffers, this is up to ~2GB of buffers.
DEFINE_Int32(max_free_io_buffers, "128");
-// The probing algorithm of partitioned hash table.
-// Enable quadratic probing hash table
-DEFINE_Bool(enable_quadratic_probing, "false");
-
// for pprof
DEFINE_String(pprof_profile_dir, "${DORIS_HOME}/log");
// for jeprofile in jemalloc
@@ -558,8 +552,6 @@ DEFINE_mBool(enable_token_check, "true");
// to open/close system metrics
DEFINE_Bool(enable_system_metrics, "true");
-DEFINE_mBool(enable_prefetch, "true");
-
// Number of cores Doris will used, this will effect only when it's greater than 0.
// Otherwise, Doris will use all cores returned from "/proc/cpuinfo".
DEFINE_Int32(num_cores, "0");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index ea44d718bc5..79704aa0e9c 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -307,8 +307,6 @@ DECLARE_mInt32(doris_max_scan_key_num);
// the max number of push down values of a single column.
// if exceed, no conditions will be pushed down for that column.
DECLARE_mInt32(max_pushdown_conditions_per_column);
-// return_row / total_row
-DECLARE_mInt32(doris_max_pushdown_conjuncts_return_rate);
// (Advanced) Maximum size of per-query receive-side buffer
DECLARE_mInt32(exchg_node_buffer_size_bytes);
@@ -592,10 +590,6 @@ DECLARE_Int32(min_buffer_size); // 1024, The minimum read buffer size (in bytes)
// With 1024B through 8MB buffers, this is up to ~2GB of buffers.
DECLARE_Int32(max_free_io_buffers);
-// The probing algorithm of partitioned hash table.
-// Enable quadratic probing hash table
-DECLARE_Bool(enable_quadratic_probing);
-
// for pprof
DECLARE_String(pprof_profile_dir);
// for jeprofile in jemalloc
@@ -609,8 +603,6 @@ DECLARE_mBool(enable_token_check);
// to open/close system metrics
DECLARE_Bool(enable_system_metrics);
-DECLARE_mBool(enable_prefetch);
-
// Number of cores Doris will used, this will effect only when it's greater than 0.
// Otherwise, Doris will use all cores returned from "/proc/cpuinfo".
DECLARE_Int32(num_cores);
diff --git a/docs/en/docs/admin-manual/config/be-config.md b/docs/en/docs/admin-manual/config/be-config.md
index 3f00e62d38b..82f54818f6e 100644
--- a/docs/en/docs/admin-manual/config/be-config.md
+++ b/docs/en/docs/admin-manual/config/be-config.md
@@ -343,12 +343,6 @@ There are two ways to configure BE configuration items:
* Description: Follow up query requests create threads dynamically, with a maximum of 512 threads created.
* Default value: 2048
-#### `doris_max_pushdown_conjuncts_return_rate`
-
-* Type: int32
-* Description: When BE performs HashJoin, it will adopt a dynamic partitioning method to push the join condition to OlapScanner. When the data scanned by OlapScanner is larger than 32768 rows, BE will check the filter condition. If the filter rate of the filter condition is lower than this configuration, Doris will stop using the dynamic partition clipping condition for data filtering.
-* Default value: 90
-
#### `doris_max_scan_key_num`
* Type: int
@@ -397,18 +391,6 @@ There are two ways to configure BE configuration items:
* Description: Max thread number of Remote scanner thread pool. Remote scanner thread pool is used for scan task of all external data sources.
* Default: 512
-#### `enable_prefetch`
-
-* Type: bool
-* Description: When using PartitionedHashTable for aggregation and join calculations, whether to perform HashBucket prefetch. Recommended to be set to true
-* Default value: true
-
-#### `enable_quadratic_probing`
-
-* Type: bool
-* Description: When a Hash conflict occurs when using PartitionedHashTable, enable to use the square detection method to resolve the Hash conflict. If the value is false, linear detection is used to resolve the Hash conflict. For the square detection method, please refer to: [quadratic_probing](https://en.wikipedia.org/wiki/Quadratic_probing)
-* Default value: true
-
#### `exchg_node_buffer_size_bytes`
* Type: int32
diff --git a/docs/zh-CN/docs/admin-manual/config/be-config.md b/docs/zh-CN/docs/admin-manual/config/be-config.md
index dc8418d548b..bda5dce7e50 100644
--- a/docs/zh-CN/docs/admin-manual/config/be-config.md
+++ b/docs/zh-CN/docs/admin-manual/config/be-config.md
@@ -354,12 +354,6 @@ BE 重启后该配置将失效。如果想持久化修改结果，使用如下
* 描述：后续查询请求动态创建线程，最大创建512个线程。
* 默认值：2048
-#### `doris_max_pushdown_conjuncts_return_rate`
-
-* 类型：int32
-* 描述：BE在进行HashJoin时，会采取动态分区裁剪的方式将join条件下推到OlapScanner上。当OlapScanner扫描的数据大于32768行时，BE会进行过滤条件检查，如果该过滤条件的过滤率低于该配置，则Doris会停止使用动态分区裁剪的条件进行数据过滤。
-* 默认值：90
-
#### `doris_max_scan_key_num`
* 类型：int
@@ -408,18 +402,6 @@ BE 重启后该配置将失效。如果想持久化修改结果，使用如下
* 描述：Remote scanner thread pool 的最大线程数。Remote scanner thread pool 用于除内表外的所有 scan 任务的执行。
* 默认值：512
-#### `enable_prefetch`
-
-* 类型：bool
-* 描述：当使用PartitionedHashTable进行聚合和join计算时，是否进行 HashBucket 的预取，推荐设置为true。
-* 默认值：true
-
-#### `enable_quadratic_probing`
-
-* 类型：bool
-* 描述：当使用PartitionedHashTable时发生Hash冲突时，是否采用平方探测法来解决Hash冲突。该值为false的话，则选用线性探测发来解决Hash冲突。关于平方探测法可参考：[quadratic_probing](https://en.wikipedia.org/wiki/Quadratic_probing)
-* 默认值：true
-
#### `exchg_node_buffer_size_bytes`
* 类型：int32
diff --git a/regression-test/suites/update/test_update_configs.groovy b/regression-test/suites/update/test_update_configs.groovy
index f2206e61c3b..cbce43293c1 100644
--- a/regression-test/suites/update/test_update_configs.groovy
+++ b/regression-test/suites/update/test_update_configs.groovy
@@ -38,16 +38,12 @@ suite("test_update_configs", "p0") {
if (((List<String>) ele)[0] == "disable_auto_compaction") {
disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2])
}
- if (((List<String>) ele)[0] == "enable_prefetch") {
- enablePrefetch = Boolean.parseBoolean(((List<String>) ele)[2])
- }
if (((List<String>) ele)[0] == "enable_segcompaction") {
enableSegcompaction = Boolean.parseBoolean(((List<String>) ele)[2])
}
}
- logger.info("disable_auto_compaction:${disableAutoCompaction}, enable_prefetch:${enablePrefetch}, enable_segcompaction:${enableSegcompaction}")
- curl("POST", String.format("http://%s:%s/api/update_config?%s=%s&%s=%s&%s=%s", beIp, bePort, "disable_auto_compaction", String.valueOf(!disableAutoCompaction), "enable_prefetch", String.valueOf(!enablePrefetch), "enable_segcompaction", String.valueOf(!enableSegcompaction)))
+ curl("POST", String.format("http://%s:%s/api/update_config?%s=%s&%s=%s", beIp, bePort, "disable_auto_compaction", String.valueOf(!disableAutoCompaction), "enable_segcompaction", String.valueOf(!enableSegcompaction)))
(code, out, err) = show_be_config(beIp, bePort)
@@ -61,10 +57,6 @@ suite("test_update_configs", "p0") {
logger.info("disable_auto_compaction: ${((List<String>) ele)[2]}")
assertEquals(((List<String>) ele)[2], String.valueOf(!disableAutoCompaction))
}
- if (((List<String>) ele)[0] == "enable_prefetch") {
- logger.info("enable_prefetch: ${((List<String>) ele)[3]}")
- assertEquals(((List<String>) ele)[2], String.valueOf(!enablePrefetch))
- }
if (((List<String>) ele)[0] == "enable_segcompaction") {
// enable_segcompaction is not mutable
logger.info("enable_segcompaction: ${((List<String>) ele)[3]}")
@@ -72,7 +64,7 @@ suite("test_update_configs", "p0") {
}
}
- curl("POST", String.format("http://%s:%s/api/update_config?%s=%s&%s=%s", beIp, bePort, "disable_auto_compaction", String.valueOf(disableAutoCompaction), "enable_prefetch", String.valueOf(enablePrefetch)))
+ curl("POST", String.format("http://%s:%s/api/update_config?%s=%s", beIp, bePort, "disable_auto_compaction", String.valueOf(disableAutoCompaction)))
(code, out, err) = show_be_config(beIp, bePort)
assertEquals(code, 0)
@@ -83,8 +75,5 @@ suite("test_update_configs", "p0") {
if (((List<String>) ele)[0] == "disable_auto_compaction") {
assertEquals(((List<String>) ele)[2], String.valueOf(disableAutoCompaction))
}
- if (((List<String>) ele)[0] == "enable_prefetch") {
- assertEquals(((List<String>) ele)[2], String.valueOf(enablePrefetch))
- }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 06/22: [fix](fe) Fix drop table write too many times editlog (#29966)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit e73c0f40779c720a799617c49b0ba5caee9eece9
Author: Lei Zhang <27...@users.noreply.github.com>
AuthorDate: Wed Jan 17 16:40:17 2024 +0800
[fix](fe) Fix drop table write too many times editlog (#29966)
---
.../java/org/apache/doris/datasource/InternalCatalog.java | 9 +++++++--
.../src/main/java/org/apache/doris/persist/EditLog.java | 4 +++-
.../java/org/apache/doris/statistics/AnalysisManager.java | 14 ++------------
3 files changed, 12 insertions(+), 15 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index b5e68de5b2e..3212d18c1d3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -903,11 +903,15 @@ public class InternalCatalog implements CatalogIf<Database> {
} finally {
table.writeUnlock();
}
- DropInfo info = new DropInfo(db.getId(), table.getId(), tableName, -1L, stmt.isForceDrop(), recycleTime);
- Env.getCurrentEnv().getEditLog().logDropTable(info);
+
Env.getCurrentEnv().getQueryStats().clear(Env.getCurrentEnv().getCurrentCatalog().getId(),
db.getId(), table.getId());
+
Env.getCurrentEnv().getAnalysisManager().removeTableStats(table.getId());
+
+ DropInfo info = new DropInfo(db.getId(), table.getId(), tableName, -1L, stmt.isForceDrop(), recycleTime);
+ Env.getCurrentEnv().getEditLog().logDropTable(info);
+
Env.getCurrentEnv().getMtmvService().dropTable(table);
} catch (UserException e) {
throw new DdlException(e.getMessage(), e.getMysqlErrorCode());
@@ -944,6 +948,7 @@ public class InternalCatalog implements CatalogIf<Database> {
unprotectDropTable(db, table, isForceDrop, true, recycleTime);
Env.getCurrentEnv().getQueryStats().clear(Env.getCurrentInternalCatalog().getId(), db.getId(),
tableId);
+ Env.getCurrentEnv().getAnalysisManager().removeTableStats(table.getId());
} finally {
table.writeUnlock();
db.writeUnlock();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
index fb97c02b00c..5b2ff25f54e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
@@ -1135,7 +1135,9 @@ public class EditLog {
break;
}
case OperationType.OP_DELETE_TABLE_STATS: {
- env.getAnalysisManager().replayTableStatsDeletion((TableStatsDeletionLog) journal.getData());
+ long tableId = ((TableStatsDeletionLog) journal.getData()).id;
+ LOG.info("replay delete table stat tableId: {}", tableId);
+ Env.getCurrentEnv().getAnalysisManager().removeTableStats(tableId);
break;
}
case OperationType.OP_ALTER_MTMV: {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index f56e800e83d..00e858c1536 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -46,7 +46,6 @@ import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.persist.AnalyzeDeletionLog;
-import org.apache.doris.persist.TableStatsDeletionLog;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowResultSet;
@@ -1015,17 +1014,8 @@ public class AnalysisManager implements Writable {
}
}
- public void removeTableStats(long tblId) {
- if (!idToTblStats.containsKey(tblId)) {
- return;
- }
- TableStatsDeletionLog log = new TableStatsDeletionLog(tblId);
- Env.getCurrentEnv().getEditLog().logDeleteTableStats(log);
- replayTableStatsDeletion(log);
- }
-
- public void replayTableStatsDeletion(TableStatsDeletionLog log) {
- idToTblStats.remove(log.id);
+ public void removeTableStats(long tableId) {
+ idToTblStats.remove(tableId);
}
public ColStatsMeta findColStatsMeta(long tblId, String colName) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 04/22: [Feature](materialized-view) support match function with alias in materialized-view (#30025)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit b0c49024cb72aa8595c0190876e70e899c24e70f
Author: Pxl <px...@qq.com>
AuthorDate: Wed Jan 17 14:33:38 2024 +0800
[Feature](materialized-view) support match function with alias in materialized-view (#30025)
support match function with alias in materialized-view
---
.../java/org/apache/doris/analysis/Analyzer.java | 2 +
.../org/apache/doris/analysis/FunctionName.java | 4 ++
.../org/apache/doris/rewrite/FunctionAlias.java | 59 ++++++++++++++++++++++
gensrc/script/doris_builtins_functions.py | 2 +-
.../data/mv_p0/test_substr/test_substr.out | 4 ++
.../suites/mv_p0/test_substr/test_substr.groovy | 56 ++++++++++++++++++++
6 files changed, 126 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
index 5dcb9c193e2..b41666820b5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
@@ -56,6 +56,7 @@ import org.apache.doris.rewrite.ExprRewriteRule;
import org.apache.doris.rewrite.ExprRewriter;
import org.apache.doris.rewrite.ExtractCommonFactorsRule;
import org.apache.doris.rewrite.FoldConstantsRule;
+import org.apache.doris.rewrite.FunctionAlias;
import org.apache.doris.rewrite.InferFiltersRule;
import org.apache.doris.rewrite.MatchPredicateRule;
import org.apache.doris.rewrite.NormalizeBinaryPredicatesRule;
@@ -458,6 +459,7 @@ public class Analyzer {
rules.add(MatchPredicateRule.INSTANCE);
rules.add(EliminateUnnecessaryFunctions.INSTANCE);
rules.add(ElementAtToSlotRefRule.INSTANCE);
+ rules.add(FunctionAlias.INSTANCE);
List<ExprRewriteRule> onceRules = Lists.newArrayList();
onceRules.add(ExtractCommonFactorsRule.INSTANCE);
onceRules.add(InferFiltersRule.INSTANCE);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionName.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionName.java
index 510af96e4a8..cb243ef03e3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionName.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionName.java
@@ -104,6 +104,10 @@ public class FunctionName implements Writable {
this.db = db;
}
+ public void setFn(String fn) {
+ this.fn = fn;
+ }
+
public String getFunction() {
return fn;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/FunctionAlias.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/FunctionAlias.java
new file mode 100644
index 00000000000..9e9c6dc9f75
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/FunctionAlias.java
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.rewrite;
+
+import org.apache.doris.analysis.Analyzer;
+import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.FunctionCallExpr;
+import org.apache.doris.common.AnalysisException;
+
+import com.google.common.collect.ImmutableMap;
+
+import java.util.Map;
+
+/**
+ * Change functio name to function class name on nereids
+ * alias list: catalog/BuiltinScalarFunctions.java
+ */
+public final class FunctionAlias implements ExprRewriteRule {
+ public static ExprRewriteRule INSTANCE = new FunctionAlias();
+
+ static final Map<String, String> aliasToName = ImmutableMap.<String, String>builder()
+ .put("array_size", "cardinality").put("size", "cardinality").put("ceiling", "ceil")
+ .put("char_length", "character_length").put("curdate", "current_date").put("curtime", "current_time")
+ .put("schema", "database").put("day", "dayofmonth").put("date_add", "days_add").put("adddate", "days_add")
+ .put("date_sub", "days_sub").put("subdate", "days_sub").put("inet_ntoa", "ipv4_num_to_string")
+ .put("inet_aton", "ipv4_string_to_num").put("inet6_ntoa", "ipv6_num_to_string")
+ .put("inet6_aton", "ipv6_string_to_num").put("lcase", "lower").put("add_months", "months_add")
+ .put("current_timestamp", "now").put("localtime", "now").put("localtimestamp", "now").put("ifnull", "nvl")
+ .put("rand", "random").put("sha", "sha1").put("substr", "substring").put("ucase", "upper").build();
+
+ @Override
+ public Expr apply(Expr expr, Analyzer analyzer, ExprRewriter.ClauseType clauseType) throws AnalysisException {
+ if (!(expr instanceof FunctionCallExpr)) {
+ return expr;
+ }
+ FunctionCallExpr functionCall = (FunctionCallExpr) expr;
+ if (aliasToName.containsKey(functionCall.getFnName().getFunction())) {
+ FunctionCallExpr result = (FunctionCallExpr) functionCall.clone();
+ result.getFnName().setFn(aliasToName.get(functionCall.getFnName().getFunction()));
+ return result;
+ }
+ return expr;
+ }
+}
diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py
index 4f4c2f37b8e..0ca8dfad354 100644
--- a/gensrc/script/doris_builtins_functions.py
+++ b/gensrc/script/doris_builtins_functions.py
@@ -75,7 +75,7 @@ visible_functions = {
"map": [
[['map'], 'MAP<K, V>', ['K', 'V', '...'], 'ALWAYS_NOT_NULLABLE', ['K', 'V']],
[['element_at', '%element_extract%'], 'V', ['MAP<K, V>', 'K'], 'ALWAYS_NULLABLE', ['K', 'V']],
- [['size', 'map_size'], 'BIGINT', ['MAP<K, V>'], '', ['K', 'V']],
+ [['size', 'map_size', 'cardinality'], 'BIGINT', ['MAP<K, V>'], '', ['K', 'V']],
[['map_contains_key'], 'BOOLEAN', ['MAP<K, V>', 'K'], 'CUSTOM', ['K', 'V']],
[['map_contains_value'], 'BOOLEAN', ['MAP<K, V>', 'V'], 'CUSTOM', ['K', 'V']],
#[['map_contains_key_like'], 'BOOLEAN', ['MAP<K, V>', 'K'], '', ['K', 'V']],
diff --git a/regression-test/data/mv_p0/test_substr/test_substr.out b/regression-test/data/mv_p0/test_substr/test_substr.out
new file mode 100644
index 00000000000..8ebcba7eec0
--- /dev/null
+++ b/regression-test/data/mv_p0/test_substr/test_substr.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_mv --
+\N \N
+
diff --git a/regression-test/suites/mv_p0/test_substr/test_substr.groovy b/regression-test/suites/mv_p0/test_substr/test_substr.groovy
new file mode 100644
index 00000000000..5397bac6dc9
--- /dev/null
+++ b/regression-test/suites/mv_p0/test_substr/test_substr.groovy
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite ("test_substr") {
+ sql """set enable_nereids_planner=true"""
+ sql """SET enable_fallback_to_original_planner=false"""
+ sql """ drop table if exists dwd;"""
+
+ sql """
+ CREATE TABLE `dwd` (
+ `id` bigint(20) NULL COMMENT 'id',
+ `created_at` datetime NULL,
+ `dt` date NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 10
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ sql """insert into dwd(id) values(1);"""
+
+ createMV ("""
+ create materialized view dwd_mv as
+ SELECT
+ substr(created_at,1,10) as statistic_date,
+ max(dt) as dt
+ FROM dwd
+ group by substr(created_at,1,10);
+ """)
+
+ sql """insert into dwd(id) values(2);"""
+
+ explain {
+ sql("SELECT substr(created_at,1,10) as statistic_date, max(dt) as dt FROM dwd group by substr(created_at,1,10);")
+ contains "(dwd_mv)"
+ }
+ qt_select_mv "SELECT substr(created_at,1,10) as statistic_date, max(dt) as dt FROM dwd group by substr(created_at,1,10);"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 13/22: [Improve](config)delete confused config for nested complex type (#29988)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit d1acd42db4d1ae010c4e0b3d0fd033ac6884ac96
Author: amory <wa...@selectdb.com>
AuthorDate: Wed Jan 17 17:41:40 2024 +0800
[Improve](config)delete confused config for nested complex type (#29988)
---
.../main/java/org/apache/doris/common/Config.java | 11 -----
.../java/org/apache/doris/analysis/TypeDef.java | 11 -----
.../plans/commands/info/ColumnDefinition.java | 14 ------
...ocal_tvf_with_complex_type_insertinto_doris.out | 3 --
.../three_level_nestedtypes_with_s3data.groovy | 2 +-
.../two_level_nestedtypes_with_s3data.groovy | 1 -
.../suites/datatype_p0/nested_types/load.groovy | 2 +-
.../map_functions/test_basic_map_function.groovy | 1 -
.../query/test_nested_type_with_count.groovy | 1 -
...st_nested_types_insert_into_with_literal.groovy | 1 -
.../test_nested_types_insert_into_with_s3.groovy | 1 -
...test_nestedtypes_csv_insert_into_with_s3.groovy | 1 -
.../test_nestedtypes_insert_into_select.groovy | 1 -
...est_nestedtypes_json_insert_into_with_s3.groovy | 1 -
...l_tvf_with_complex_type_insertinto_doris.groovy | 2 +-
.../test_load_with_map_nested_array.groovy | 1 -
.../aggregate/map_agg_nested_insert_doris.groovy | 2 +-
.../show/test_nested_complex_switch.groovy | 56 ----------------------
18 files changed, 4 insertions(+), 108 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index f5865e1b7ba..01fd3596895 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2089,17 +2089,6 @@ public class Config extends ConfigBase {
@ConfField(mutable = true)
public static boolean disable_datev1 = true;
- /**
- * Now we not fully support array/struct/map nesting complex type in many situation,
- * so just disable creating nesting complex data type when create table.
- * We can make it able after we fully support
- */
- @ConfField(mutable = true, masterOnly = true, description = {
- "当前默认设置为 true，不支持建表时创建复杂类型(array/struct/map)嵌套复杂类型, 仅支持array类型自身嵌套。",
- "Now default set to true, not support create complex type(array/struct/map) nested complex type "
- + "when we create table, only support array type nested array"})
- public static boolean disable_nested_complex_type = true;
-
/*
* This variable indicates the number of digits by which to increase the scale
* of the result of division operations performed with the `/` operator. The
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
index c7199b82b28..2e8c1291b8b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
@@ -28,7 +28,6 @@ import org.apache.doris.catalog.StructField;
import org.apache.doris.catalog.StructType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Config;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.thrift.TColumnDesc;
import org.apache.doris.thrift.TPrimitiveType;
@@ -130,18 +129,10 @@ public class TypeDef implements ParseNode {
Type itemType = ((ArrayType) type).getItemType();
if (itemType instanceof ScalarType) {
analyzeNestedType(type, (ScalarType) itemType);
- } else if (Config.disable_nested_complex_type && !(itemType instanceof ArrayType)) {
- // now we can array nesting array
- throw new AnalysisException("Unsupported data type: ARRAY<" + itemType.toSql() + ">");
}
}
if (type.isMapType()) {
MapType mt = (MapType) type;
- if (Config.disable_nested_complex_type && (!(mt.getKeyType() instanceof ScalarType)
- || !(mt.getValueType() instanceof ScalarType))) {
- throw new AnalysisException("Unsupported data type: MAP<" + mt.getKeyType().toSql() + ","
- + mt.getValueType().toSql() + ">");
- }
if (mt.getKeyType() instanceof ScalarType) {
analyzeNestedType(type, (ScalarType) mt.getKeyType());
}
@@ -160,8 +151,6 @@ public class TypeDef implements ParseNode {
throw new AnalysisException("Duplicate field name "
+ field.getName() + " in struct " + type.toSql());
}
- } else if (Config.disable_nested_complex_type) {
- throw new AnalysisException("Unsupported field type: " + fieldType.toSql() + " for STRUCT");
}
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
index 5832a0534c0..bf85d405503 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
@@ -24,7 +24,6 @@ import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
-import org.apache.doris.common.Config;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.types.ArrayType;
import org.apache.doris.nereids.types.BigIntType;
@@ -392,21 +391,11 @@ public class ColumnDefinition {
Type itemType = ((org.apache.doris.catalog.ArrayType) catalogType).getItemType();
if (itemType instanceof ScalarType) {
validateNestedType(catalogType, (ScalarType) itemType);
- } else if (Config.disable_nested_complex_type
- && !(itemType instanceof org.apache.doris.catalog.ArrayType)) {
- // now we can array nesting array
- throw new AnalysisException(
- "Unsupported data type: ARRAY<" + itemType.toSql() + ">");
}
}
if (catalogType.isMapType()) {
org.apache.doris.catalog.MapType mt =
(org.apache.doris.catalog.MapType) catalogType;
- if (Config.disable_nested_complex_type && (!(mt.getKeyType() instanceof ScalarType)
- || !(mt.getValueType() instanceof ScalarType))) {
- throw new AnalysisException("Unsupported data type: MAP<"
- + mt.getKeyType().toSql() + "," + mt.getValueType().toSql() + ">");
- }
if (mt.getKeyType() instanceof ScalarType) {
validateNestedType(catalogType, (ScalarType) mt.getKeyType());
}
@@ -426,9 +415,6 @@ public class ColumnDefinition {
throw new AnalysisException("Duplicate field name " + field.getName()
+ " in struct " + catalogType.toSql());
}
- } else if (Config.disable_nested_complex_type) {
- throw new AnalysisException(
- "Unsupported field type: " + fieldType.toSql() + " for STRUCT");
}
}
}
diff --git a/regression-test/data/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.out b/regression-test/data/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.out
index ebc6fbe02b8..8e99380d460 100644
--- a/regression-test/data/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.out
+++ b/regression-test/data/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.out
@@ -2,9 +2,6 @@
-- !sql --
0
--- !sql --
-0
-
-- !sql --
2 {1:[0.35815922932906263, 0.0011899152357573994, 0.28749219850167373, 0.93512930168283781, 0.1552584991620739, 0.73308976093672584, 0.52815960653805338, 0.92936404769642733, 0.89084215295591418, 0.21986459138832559], 2:[0.25076205844319044, 0.54003619330849928, 0.70661164863002113, 0.99472899095144263, 0.078314941019622886, 0.44206845606243961, 0.30857433265707379, 0.048661247184825784, 0.76954870938240083, 0.27253204080482074], 3:[0.687619207224633, 0.81614795840545351, 0.8866348164721 [...]
diff --git a/regression-test/suites/datatype_p0/nested_types/base_cases/three_level_nestedtypes_with_s3data.groovy b/regression-test/suites/datatype_p0/nested_types/base_cases/three_level_nestedtypes_with_s3data.groovy
index 4258c30e1d4..66897b161d3 100644
--- a/regression-test/suites/datatype_p0/nested_types/base_cases/three_level_nestedtypes_with_s3data.groovy
+++ b/regression-test/suites/datatype_p0/nested_types/base_cases/three_level_nestedtypes_with_s3data.groovy
@@ -20,7 +20,7 @@ import org.apache.commons.lang3.StringUtils
suite("three_level_nestedtypes_with_s3data") {
sql """set enable_nereids_planner=false"""
sql """ set enable_fallback_to_original_planner=true;"""
- sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false');"""
+
// this test case aim to test one-level nested type with s3 data
diff --git a/regression-test/suites/datatype_p0/nested_types/base_cases/two_level_nestedtypes_with_s3data.groovy b/regression-test/suites/datatype_p0/nested_types/base_cases/two_level_nestedtypes_with_s3data.groovy
index c00ccfcf7d6..d4b701b94b8 100644
--- a/regression-test/suites/datatype_p0/nested_types/base_cases/two_level_nestedtypes_with_s3data.groovy
+++ b/regression-test/suites/datatype_p0/nested_types/base_cases/two_level_nestedtypes_with_s3data.groovy
@@ -20,7 +20,6 @@ import org.apache.commons.lang3.StringUtils
suite("two_level_nestedtypes_with_s3data") {
sql """set enable_nereids_planner=false"""
sql """ set enable_fallback_to_original_planner=true;"""
- sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false');"""
// this test case aim to test one-level nested type with s3 data
diff --git a/regression-test/suites/datatype_p0/nested_types/load.groovy b/regression-test/suites/datatype_p0/nested_types/load.groovy
index 3ceffc30cd4..2bf559425d1 100644
--- a/regression-test/suites/datatype_p0/nested_types/load.groovy
+++ b/regression-test/suites/datatype_p0/nested_types/load.groovy
@@ -17,7 +17,7 @@
suite("load") {
// ddl begin
- sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""
+
sql """set enable_nereids_planner=false"""
def dataFile = """test_scalar_types_100.csv"""
diff --git a/regression-test/suites/datatype_p0/nested_types/query/map_functions/test_basic_map_function.groovy b/regression-test/suites/datatype_p0/nested_types/query/map_functions/test_basic_map_function.groovy
index 88993c5aeb0..1b4a4e8a919 100644
--- a/regression-test/suites/datatype_p0/nested_types/query/map_functions/test_basic_map_function.groovy
+++ b/regression-test/suites/datatype_p0/nested_types/query/map_functions/test_basic_map_function.groovy
@@ -16,7 +16,6 @@
// under the License.
suite("test_basic_map_function", "p0") {
- sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false'); """
sql """set enable_nereids_planner=false"""
// ============ sum(map-value) ============
qt_sql """ SELECT "sum-map-value" """
diff --git a/regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_count.groovy b/regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_count.groovy
index 2b5bf97795e..1bc8285257e 100644
--- a/regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_count.groovy
+++ b/regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_count.groovy
@@ -20,7 +20,6 @@ import org.apache.commons.lang3.StringUtils
suite("test_nested_type_with_count") {
// this test case aim to test nested type with old planner
sql """set enable_nereids_planner=false"""
- sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false');"""
def table_names = ["test_array_one_level", "test_map_one_level", "test_struct_one_level"]
diff --git a/regression-test/suites/datatype_p0/nested_types/query/test_nested_types_insert_into_with_literal.groovy b/regression-test/suites/datatype_p0/nested_types/query/test_nested_types_insert_into_with_literal.groovy
index da3262858ae..08e794b23f3 100644
--- a/regression-test/suites/datatype_p0/nested_types/query/test_nested_types_insert_into_with_literal.groovy
+++ b/regression-test/suites/datatype_p0/nested_types/query/test_nested_types_insert_into_with_literal.groovy
@@ -25,7 +25,6 @@ suite("test_nested_types_insert_into_with_literal", "p0") {
// old planner does not support cast empty
sql 'set enable_nereids_planner=true'
sql 'set enable_fallback_to_original_planner=false'
- sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""
def table_names = [
"two_level_array_array_a",
diff --git a/regression-test/suites/datatype_p0/nested_types/query/test_nested_types_insert_into_with_s3.groovy b/regression-test/suites/datatype_p0/nested_types/query/test_nested_types_insert_into_with_s3.groovy
index 2b59e76aa29..fc2d3d25988 100644
--- a/regression-test/suites/datatype_p0/nested_types/query/test_nested_types_insert_into_with_s3.groovy
+++ b/regression-test/suites/datatype_p0/nested_types/query/test_nested_types_insert_into_with_s3.groovy
@@ -24,7 +24,6 @@ suite("test_nested_types_insert_into_with_s3", "p0") {
sql 'use regression_test_datatype_p0_nested_types'
sql 'set enable_nereids_planner=false'
sql 'set max_allowed_packet=4194304'
- sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""
String ak = getS3AK()
String sk = getS3SK()
diff --git a/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_csv_insert_into_with_s3.groovy b/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_csv_insert_into_with_s3.groovy
index ab9dff9ffb2..40f76575bcf 100644
--- a/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_csv_insert_into_with_s3.groovy
+++ b/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_csv_insert_into_with_s3.groovy
@@ -25,7 +25,6 @@ suite("test_nestedtypes_csv_insert_into_with_s3", "p0") {
sql 'set enable_nereids_planner=false'
sql 'set max_allowed_packet=4194304'
sql 'set topn_opt_limit_threshold=10000'
- sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""
String ak = getS3AK()
String sk = getS3SK()
diff --git a/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy b/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy
index a74687af64e..bc21c18a0a9 100644
--- a/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy
+++ b/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy
@@ -22,7 +22,6 @@ import org.codehaus.groovy.runtime.IOGroovyMethods
suite("test_nestedtypes_insert_into_select", "p0") {
sql "set enable_nereids_planner=false"
- sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""
// create array struct
sql "DROP TABLE IF EXISTS ast;"
diff --git a/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_json_insert_into_with_s3.groovy b/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_json_insert_into_with_s3.groovy
index fb83de7a340..66d5c6bc5ee 100644
--- a/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_json_insert_into_with_s3.groovy
+++ b/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_json_insert_into_with_s3.groovy
@@ -25,7 +25,6 @@ suite("test_nestedtypes_json_insert_into_with_s3", "p0") {
sql 'set enable_nereids_planner=false'
sql 'set max_allowed_packet=4194304'
sql 'set topn_opt_limit_threshold=10000'
- sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""
String ak = getS3AK()
String sk = getS3SK()
diff --git a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy
index 04feab78ddb..c8bd8a8c2d6 100644
--- a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy
@@ -43,7 +43,7 @@ suite("test_local_tvf_with_complex_type_insertinto_doris", "p0") {
}
}
- qt_sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""
+
// create doris table
sql """ DROP TABLE IF EXISTS ${table_name} """
diff --git a/regression-test/suites/load_p0/stream_load/test_load_with_map_nested_array.groovy b/regression-test/suites/load_p0/stream_load/test_load_with_map_nested_array.groovy
index 7e7e676bf3a..312905bc868 100644
--- a/regression-test/suites/load_p0/stream_load/test_load_with_map_nested_array.groovy
+++ b/regression-test/suites/load_p0/stream_load/test_load_with_map_nested_array.groovy
@@ -17,7 +17,6 @@
suite("test_load_with_map_nested_array", "p0") {
def tableName = "test_load_with_map_nested_array"
- sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false'); """
sql """ DROP TABLE IF EXISTS ${tableName} """
sql """
diff --git a/regression-test/suites/query_p0/aggregate/map_agg_nested_insert_doris.groovy b/regression-test/suites/query_p0/aggregate/map_agg_nested_insert_doris.groovy
index a4b0410cba3..8d4ae13b1cc 100644
--- a/regression-test/suites/query_p0/aggregate/map_agg_nested_insert_doris.groovy
+++ b/regression-test/suites/query_p0/aggregate/map_agg_nested_insert_doris.groovy
@@ -20,7 +20,7 @@ suite("map_agg_nested_insert_doris", "p0") {
def tb_doris = "test_map_agg_nested_insert_target"
sql "DROP TABLE IF EXISTS `${tb_base}`;"
sql "DROP TABLE IF EXISTS `${tb_doris}`;"
- sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false'); """
+
sql """
CREATE TABLE `${tb_base}` (
diff --git a/regression-test/suites/query_p0/show/test_nested_complex_switch.groovy b/regression-test/suites/query_p0/show/test_nested_complex_switch.groovy
index af39d8a9c32..9b06a00eb42 100644
--- a/regression-test/suites/query_p0/show/test_nested_complex_switch.groovy
+++ b/regression-test/suites/query_p0/show/test_nested_complex_switch.groovy
@@ -137,63 +137,7 @@ suite("test_nested_complex_switch", "query") {
sql "DROP TABLE IF EXISTS ${testTable_m}"
sql "DROP TABLE IF EXISTS ${testTable_a}"
sql "DROP TABLE IF EXISTS ${testTable_s}"
- sql "ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'true')"
- // map
- test {
- sql sql_m_s
- exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: MAP<TEXT,STRUCT<f1:TINYINT>>"
- }
-
- test {
- sql sql_m_a
- exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: MAP<ARRAY<INT>,TEXT>"
- }
-
- test {
- sql sql_m_m
- exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: MAP<TEXT,MAP<TEXT,INT>>"
- }
-
- // array
- test {
- sql sql_a_s
- exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: ARRAY<STRUCT<f1:TINYINT>>"
- }
-
-
- test {
- sql sql_a_m
- exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: ARRAY<MAP<TEXT,INT>>"
- }
-
- // struct
- test {
- sql sql_s_s
- exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported"
- }
-
- test {
- sql sql_s_a
- exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported"
- }
-
- test {
- sql sql_s_m
- exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported"
- }
-
- } finally {
- try_sql("DROP TABLE IF EXISTS ${testTable_m}")
- try_sql("DROP TABLE IF EXISTS ${testTable_a}")
- try_sql("DROP TABLE IF EXISTS ${testTable_s}")
- }
-
- try {
- sql "DROP TABLE IF EXISTS ${testTable_m}"
- sql "DROP TABLE IF EXISTS ${testTable_a}"
- sql "DROP TABLE IF EXISTS ${testTable_s}"
- sql "ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"
// map
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 22/22: [improvement](fe) skip attach profile during send fragment (#30085)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit e7c6c12a65a62b65a30428f54c081dcb42c6f509
Author: yiguolei <67...@qq.com>
AuthorDate: Thu Jan 18 10:00:53 2024 +0800
[improvement](fe) skip attach profile during send fragment (#30085)
Sometime attach profile may cost a lot of time in large machine with many instances.
---------
Co-authored-by: yiguolei <yi...@gmail.com>
---
.../src/main/java/org/apache/doris/qe/Coordinator.java | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 2f58e560740..586a27ee103 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -233,7 +233,6 @@ public class Coordinator implements CoordInterface {
// Input parameter
private long jobId = -1; // job which this task belongs to
private TUniqueId queryId;
- private final boolean needReport;
// parallel execute
private final TUniqueId nextInstanceId;
@@ -324,7 +323,6 @@ public class Coordinator implements CoordInterface {
} else {
this.queryGlobals.setTimeZone(context.getSessionVariable().getTimeZone());
}
- this.needReport = context.getSessionVariable().enableProfile();
this.nextInstanceId = new TUniqueId();
nextInstanceId.setHi(queryId.hi);
nextInstanceId.setLo(queryId.lo + 1);
@@ -348,7 +346,6 @@ public class Coordinator implements CoordInterface {
this.queryGlobals.setTimeZone(timezone);
this.queryGlobals.setLoadZeroTolerance(loadZeroTolerance);
this.queryOptions.setBeExecVersion(Config.be_exec_version);
- this.needReport = true;
this.nextInstanceId = new TUniqueId();
nextInstanceId.setHi(queryId.hi);
nextInstanceId.setLo(queryId.lo + 1);
@@ -845,8 +842,9 @@ public class Coordinator implements CoordInterface {
}
waitRpc(futures, this.timeoutDeadline - System.currentTimeMillis(), "send execution start");
}
-
- attachInstanceProfileToFragmentProfile();
+ if (context != null && context.getSessionVariable().enableProfile()) {
+ attachInstanceProfileToFragmentProfile();
+ }
} finally {
unlock();
}
@@ -988,8 +986,9 @@ public class Coordinator implements CoordInterface {
}
waitPipelineRpc(futures, this.timeoutDeadline - System.currentTimeMillis(), "send execution start");
}
-
- attachInstanceProfileToFragmentProfile();
+ if (context != null && context.getSessionVariable().enableProfile()) {
+ attachInstanceProfileToFragmentProfile();
+ }
} finally {
unlock();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 10/22: [function](char) change char function behaviour same with mysql (#30034)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit e894911cdaa72a0f89e1f6ab0a42522a85f498f0
Author: zhangstar333 <87...@users.noreply.github.com>
AuthorDate: Wed Jan 17 17:15:06 2024 +0800
[function](char) change char function behaviour same with mysql (#30034)
select char(0) = '\0';
should return true;
---
be/src/vec/functions/function_string.h | 6 +++---
.../string_functions/test_string_function.out | Bin 3360 -> 3418 bytes
.../string_functions/test_string_function.groovy | 2 ++
3 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/be/src/vec/functions/function_string.h b/be/src/vec/functions/function_string.h
index 79b5205640a..0a4b3432015 100644
--- a/be/src/vec/functions/function_string.h
+++ b/be/src/vec/functions/function_string.h
@@ -3243,7 +3243,7 @@ private:
void integer_to_char_(int line_num, const int* num, ColumnString::Chars& chars,
IColumn::Offsets& offsets) const {
if (0 == *num) {
- chars.push_back(' ');
+ chars.push_back('\0');
offsets[line_num] = offsets[line_num - 1] + 1;
return;
}
@@ -3257,7 +3257,7 @@ private:
}
offsets[line_num] = offsets[line_num - 1] + k + 1;
for (; k >= 0; --k) {
- chars.push_back(bytes[k] ? bytes[k] : ' ');
+ chars.push_back(bytes[k] ? bytes[k] : '\0');
}
#else
int k = 0;
@@ -3268,7 +3268,7 @@ private:
}
offsets[line_num] = offsets[line_num - 1] + 4 - k;
for (; k < 4; ++k) {
- chars.push_back(bytes[k] ? bytes[k] : ' ');
+ chars.push_back(bytes[k] ? bytes[k] : '\0');
}
#endif
}
diff --git a/regression-test/data/query_p0/sql_functions/string_functions/test_string_function.out b/regression-test/data/query_p0/sql_functions/string_functions/test_string_function.out
index b51fb32d615..118c58c4910 100644
Binary files a/regression-test/data/query_p0/sql_functions/string_functions/test_string_function.out and b/regression-test/data/query_p0/sql_functions/string_functions/test_string_function.out differ
diff --git a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy
index 4f9faac47f9..cee30808b84 100644
--- a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy
@@ -280,5 +280,7 @@ suite("test_string_function", "arrow_flight_sql") {
qt_sql_func_char6 """ select char(k1) from test_function_char order by k1; """
qt_sql_func_char7 """ select char(k1, k2, k3, k4) from test_function_char order by k1, k2, k3, k4; """
qt_sql_func_char8 """ select char(k1, k2, k3, k4, 65) from test_function_char order by k1, k2, k3, k4; """
+ qt_sql_func_char9 """ select char(0) = ' '; """
+ qt_sql_func_char10 """ select char(0) = '\0'; """
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 14/22: [feature](statistics)support statistics for iceberg/paimon/hudi table (#29868)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 40d5e664d6def7bf0cccc85e008bdea8bac89715
Author: wuwenchi <wu...@hotmail.com>
AuthorDate: Wed Jan 17 17:43:05 2024 +0800
[feature](statistics)support statistics for iceberg/paimon/hudi table (#29868)
---
.../docker-compose/iceberg/iceberg.env | 2 +-
.../docker-compose/iceberg/iceberg.yaml.tpl | 2 +
.../catalog/external/IcebergExternalTable.java | 9 +
.../catalog/external/PaimonExternalTable.java | 9 +
.../apache/doris/common/proc/CatalogsProcDir.java | 10 +-
...AnalysisTask.java => ExternalAnalysisTask.java} | 173 ++------------
.../apache/doris/statistics/HMSAnalysisTask.java | 252 ++-------------------
.../doris/statistics/util/StatisticsUtil.java | 10 +-
.../doris/statistics/HMSAnalysisTaskTest.java | 4 +-
regression-test/conf/regression-conf.groovy | 1 +
.../iceberg/test_iceberg_statistics.out | 39 ++++
.../paimon/test_paimon_statistics.out | 21 ++
.../hive/test_hive_hudi_statistics.out | 16 ++
.../pipeline/p0/conf/regression-conf.groovy | 1 +
.../tpch/tpch-sf100/conf/regression-conf.groovy | 1 +
.../plugins/plugins_get_ids_from_proc.groovy | 62 +++++
.../iceberg/test_iceberg_statistics.groovy | 57 +++++
.../paimon/test_paimon_catalog.groovy | 3 +
.../paimon/test_paimon_statistics.groovy | 47 ++++
.../external_table_p0/test_catalog_ddl.groovy | 2 +
.../hive/test_hive_hudi_statistics.groovy | 47 ++++
21 files changed, 372 insertions(+), 396 deletions(-)
diff --git a/docker/thirdparties/docker-compose/iceberg/iceberg.env b/docker/thirdparties/docker-compose/iceberg/iceberg.env
index 4cc8b42eaf9..6bebd49f437 100644
--- a/docker/thirdparties/docker-compose/iceberg/iceberg.env
+++ b/docker/thirdparties/docker-compose/iceberg/iceberg.env
@@ -21,4 +21,4 @@ SPARK_DRIVER_UI_PORT=8080
SPARK_HISTORY_UI_PORT=10000
REST_CATALOG_PORT=18181
MINIO_UI_PORT=9000
-MINIO_API_PORT=9001
+MINIO_API_PORT=19001
diff --git a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
index d7220f24376..bc217c1dd6e 100644
--- a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
+++ b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
@@ -58,6 +58,8 @@ services:
minio:
image: minio/minio
container_name: doris--minio
+ ports:
+ - ${MINIO_API_PORT}:9000
environment:
- MINIO_ROOT_USER=admin
- MINIO_ROOT_PASSWORD=password
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/IcebergExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/IcebergExternalTable.java
index 7398ff19c9e..be99e26de62 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/IcebergExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/IcebergExternalTable.java
@@ -23,7 +23,10 @@ import org.apache.doris.catalog.HiveMetaStoreClientHelper;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
+import org.apache.doris.statistics.AnalysisInfo;
+import org.apache.doris.statistics.BaseAnalysisTask;
import org.apache.doris.statistics.ColumnStatistic;
+import org.apache.doris.statistics.ExternalAnalysisTask;
import org.apache.doris.statistics.util.StatisticsUtil;
import org.apache.doris.thrift.THiveTable;
import org.apache.doris.thrift.TIcebergTable;
@@ -149,4 +152,10 @@ public class IcebergExternalTable extends ExternalTable {
() -> StatisticsUtil.getIcebergColumnStats(colName,
((IcebergExternalCatalog) catalog).getIcebergTable(dbName, name)));
}
+
+ @Override
+ public BaseAnalysisTask createAnalysisTask(AnalysisInfo info) {
+ makeSureInitialized();
+ return new ExternalAnalysisTask(info);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/PaimonExternalTable.java
index c8ea253671d..b517265df6a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/PaimonExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/PaimonExternalTable.java
@@ -21,6 +21,9 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.datasource.paimon.PaimonExternalCatalog;
+import org.apache.doris.statistics.AnalysisInfo;
+import org.apache.doris.statistics.BaseAnalysisTask;
+import org.apache.doris.statistics.ExternalAnalysisTask;
import org.apache.doris.thrift.THiveTable;
import org.apache.doris.thrift.TTableDescriptor;
import org.apache.doris.thrift.TTableType;
@@ -154,4 +157,10 @@ public class PaimonExternalTable extends ExternalTable {
+ getPaimonCatalogType());
}
}
+
+ @Override
+ public BaseAnalysisTask createAnalysisTask(AnalysisInfo info) {
+ makeSureInitialized();
+ return new ExternalAnalysisTask(info);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java
index 854b4dddc79..e6163645c28 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java
@@ -27,6 +27,7 @@ import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import org.apache.log4j.Logger;
import java.util.ArrayList;
import java.util.Collections;
@@ -37,6 +38,7 @@ import java.util.List;
* show all catalogs' info
*/
public class CatalogsProcDir implements ProcDirInterface {
+ private static final Logger LOG = Logger.getLogger(CatalogsProcDir.class);
public static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>()
.add("CatalogIds").add("CatalogName").add("DatabaseNum").add("LastUpdateTime")
.build();
@@ -90,7 +92,13 @@ public class CatalogsProcDir implements ProcDirInterface {
List<Comparable> catalogInfo = Lists.newArrayList();
catalogInfo.add(catalog.getId());
catalogInfo.add(catalog.getName());
- catalogInfo.add(catalog.getDbNames().size());
+ int size = -1;
+ try {
+ size = catalog.getDbNames().size();
+ } catch (Exception e) {
+ LOG.warn("failed to get database: ", e);
+ }
+ catalogInfo.add(size);
catalogInfo.add(TimeUtils.longToTimeString(catalog.getLastUpdateTime()));
catalogInfos.add(catalogInfo);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java
similarity index 59%
copy from fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java
copy to fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java
index fd0a4c82538..15848c013d6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java
@@ -19,15 +19,11 @@ package org.apache.doris.statistics;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
-import org.apache.doris.catalog.external.HMSExternalTable;
-import org.apache.doris.common.AnalysisException;
+import org.apache.doris.catalog.external.ExternalTable;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.Pair;
-import org.apache.doris.datasource.hive.HiveMetaStoreCache;
-import org.apache.doris.external.hive.util.HiveUtil;
import org.apache.doris.statistics.util.StatisticsUtil;
-import com.google.common.collect.Sets;
import org.apache.commons.text.StringSubstitutor;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -39,42 +35,43 @@ import java.util.Map;
import java.util.Random;
import java.util.Set;
-public class HMSAnalysisTask extends BaseAnalysisTask {
- private static final Logger LOG = LogManager.getLogger(HMSAnalysisTask.class);
+public class ExternalAnalysisTask extends BaseAnalysisTask {
+ private static final Logger LOG = LogManager.getLogger(ExternalAnalysisTask.class);
private static final String ANALYZE_TABLE_COUNT_TEMPLATE = "SELECT ROUND(COUNT(1) * ${scaleFactor}) as rowCount "
+ "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${sampleHints}";
private boolean isTableLevelTask;
private boolean isPartitionOnly;
- private HMSExternalTable table;
+ private ExternalTable table;
- public HMSAnalysisTask() {
+ // For test
+ public ExternalAnalysisTask() {
}
- public HMSAnalysisTask(AnalysisInfo info) {
+ public ExternalAnalysisTask(AnalysisInfo info) {
super(info);
isTableLevelTask = info.externalTableLevelTask;
isPartitionOnly = info.partitionOnly;
- table = (HMSExternalTable) tbl;
+ table = (ExternalTable) tbl;
}
public void doExecute() throws Exception {
if (isTableLevelTask) {
getTableStats();
} else {
- getTableColumnStats();
+ getOrdinaryColumnStats();
}
}
// For test
- protected void setTable(HMSExternalTable table) {
+ protected void setTable(ExternalTable table) {
this.table = table;
}
/**
* Get table row count
*/
- private void getTableStats() throws Exception {
+ private void getTableStats() {
Map<String, String> params = buildStatsParams(null);
List<ResultRow> columnResult =
StatisticsUtil.execStatisticQuery(new StringSubstitutor(params)
@@ -86,35 +83,8 @@ public class HMSAnalysisTask extends BaseAnalysisTask {
job.rowCountDone(this);
}
- /**
- * Get column statistics and insert the result to __internal_schema.column_statistics
- */
- protected void getTableColumnStats() throws Exception {
- if (!info.usingSqlForPartitionColumn) {
- try {
- if (isPartitionColumn()) {
- getPartitionColumnStats();
- } else {
- getHmsColumnStats();
- }
- } catch (Exception e) {
- LOG.warn("Failed to collect stats for {}col {} using metadata, "
- + "fallback to normal collection",
- isPartitionColumn() ? "partition " : "", col.getName(), e);
- /* retry using sql way! */
- getOrdinaryColumnStats();
- }
- } else {
- getOrdinaryColumnStats();
- }
- }
-
- private boolean isPartitionColumn() {
- return table.getPartitionColumns().stream().anyMatch(c -> c.getName().equals(col.getName()));
- }
-
- // Get ordinary column stats. Ordinary column means not partition column.
- private void getOrdinaryColumnStats() throws Exception {
+ // Get ordinary column stats
+ protected void getOrdinaryColumnStats() throws Exception {
StringBuilder sb = new StringBuilder();
Map<String, String> params = buildStatsParams("NULL");
params.put("min", getMinFunction());
@@ -169,122 +139,7 @@ public class HMSAnalysisTask extends BaseAnalysisTask {
runQuery(sql);
}
- // Collect the partition column stats through HMS metadata.
- // Get all the partition values and calculate the stats based on the values.
- private void getPartitionColumnStats() throws Exception {
- Set<String> partitionNames = table.getPartitionNames();
- Set<String> ndvPartValues = Sets.newHashSet();
- long numNulls = 0;
- long dataSize = 0;
- String min = null;
- String max = null;
- for (String names : partitionNames) {
- // names is like "date=20230101" for one level partition
- // and like "date=20230101/hour=12" for two level partition
- String[] parts = names.split("/");
- for (String part : parts) {
- if (part.startsWith(col.getName())) {
- String value = HiveUtil.getHivePartitionValue(part);
- // HIVE_DEFAULT_PARTITION hive partition value when the partition name is not specified.
- if (value == null || value.isEmpty() || value.equals(HiveMetaStoreCache.HIVE_DEFAULT_PARTITION)) {
- numNulls += 1;
- continue;
- }
- ndvPartValues.add(value);
- dataSize += col.getType().isStringType() ? value.length() : col.getType().getSlotSize();
- min = updateMinValue(min, value);
- max = updateMaxValue(max, value);
- }
- }
- }
- // Estimate the row count. This value is inaccurate if the table stats is empty.
- TableStatsMeta tableStatsStatus = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(table.getId());
- long count = tableStatsStatus == null ? table.estimatedRowCount() : tableStatsStatus.rowCount;
- dataSize = dataSize * count / partitionNames.size();
- numNulls = numNulls * count / partitionNames.size();
- int ndv = ndvPartValues.size();
-
- Map<String, String> params = buildStatsParams("NULL");
- params.put("row_count", String.valueOf(count));
- params.put("ndv", String.valueOf(ndv));
- params.put("null_count", String.valueOf(numNulls));
- params.put("min", StatisticsUtil.quote(min));
- params.put("max", StatisticsUtil.quote(max));
- params.put("data_size", String.valueOf(dataSize));
- StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
- String sql = stringSubstitutor.replace(ANALYZE_PARTITION_COLUMN_TEMPLATE);
- runQuery(sql);
- }
-
- // Collect the spark analyzed column stats through HMS metadata.
- private void getHmsColumnStats() throws Exception {
- TableStatsMeta tableStatsStatus = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(table.getId());
- long count = tableStatsStatus == null ? table.estimatedRowCount() : tableStatsStatus.rowCount;
-
- Map<String, String> params = buildStatsParams("NULL");
- Map<StatsType, String> statsParams = new HashMap<>();
- statsParams.put(StatsType.NDV, "ndv");
- statsParams.put(StatsType.NUM_NULLS, "null_count");
- statsParams.put(StatsType.MIN_VALUE, "min");
- statsParams.put(StatsType.MAX_VALUE, "max");
- statsParams.put(StatsType.AVG_SIZE, "avg_len");
-
- if (table.fillColumnStatistics(info.colName, statsParams, params)) {
- throw new AnalysisException("some column stats not available");
- }
-
- long dataSize = Long.valueOf(params.get("avg_len")) * count;
- params.put("row_count", String.valueOf(count));
- params.put("data_size", String.valueOf(dataSize));
-
- StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
- String sql = stringSubstitutor.replace(ANALYZE_PARTITION_COLUMN_TEMPLATE);
- runQuery(sql);
- }
-
- private String updateMinValue(String currentMin, String value) {
- if (currentMin == null) {
- return value;
- }
- if (col.getType().isFixedPointType()) {
- if (Long.parseLong(value) < Long.parseLong(currentMin)) {
- return value;
- } else {
- return currentMin;
- }
- }
- if (col.getType().isFloatingPointType() || col.getType().isDecimalV2() || col.getType().isDecimalV3()) {
- if (Double.parseDouble(value) < Double.parseDouble(currentMin)) {
- return value;
- } else {
- return currentMin;
- }
- }
- return value.compareTo(currentMin) < 0 ? value : currentMin;
- }
-
- private String updateMaxValue(String currentMax, String value) {
- if (currentMax == null) {
- return value;
- }
- if (col.getType().isFixedPointType()) {
- if (Long.parseLong(value) > Long.parseLong(currentMax)) {
- return value;
- } else {
- return currentMax;
- }
- }
- if (col.getType().isFloatingPointType() || col.getType().isDecimalV2() || col.getType().isDecimalV3()) {
- if (Double.parseDouble(value) > Double.parseDouble(currentMax)) {
- return value;
- } else {
- return currentMax;
- }
- }
- return value.compareTo(currentMax) > 0 ? value : currentMax;
- }
-
- private Map<String, String> buildStatsParams(String partId) {
+ protected Map<String, String> buildStatsParams(String partId) {
Map<String, String> commonParams = new HashMap<>();
String id = StatisticsUtil.constructId(tbl.getId(), -1);
if (partId == null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java
index fd0a4c82538..9e8be622824 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java
@@ -17,12 +17,10 @@
package org.apache.doris.statistics;
-import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.external.ExternalTable;
import org.apache.doris.catalog.external.HMSExternalTable;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.FeConstants;
-import org.apache.doris.common.Pair;
import org.apache.doris.datasource.hive.HiveMetaStoreCache;
import org.apache.doris.external.hive.util.HiveUtil;
import org.apache.doris.statistics.util.StatisticsUtil;
@@ -32,64 +30,36 @@ import org.apache.commons.text.StringSubstitutor;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import java.util.Collections;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
-import java.util.Random;
import java.util.Set;
-public class HMSAnalysisTask extends BaseAnalysisTask {
+public class HMSAnalysisTask extends ExternalAnalysisTask {
private static final Logger LOG = LogManager.getLogger(HMSAnalysisTask.class);
+ private HMSExternalTable hmsExternalTable;
- private static final String ANALYZE_TABLE_COUNT_TEMPLATE = "SELECT ROUND(COUNT(1) * ${scaleFactor}) as rowCount "
- + "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${sampleHints}";
- private boolean isTableLevelTask;
- private boolean isPartitionOnly;
- private HMSExternalTable table;
-
+ // for test
public HMSAnalysisTask() {
}
public HMSAnalysisTask(AnalysisInfo info) {
super(info);
- isTableLevelTask = info.externalTableLevelTask;
- isPartitionOnly = info.partitionOnly;
- table = (HMSExternalTable) tbl;
+ hmsExternalTable = (HMSExternalTable) tbl;
}
- public void doExecute() throws Exception {
- if (isTableLevelTask) {
- getTableStats();
- } else {
- getTableColumnStats();
- }
+ private boolean isPartitionColumn() {
+ return hmsExternalTable.getPartitionColumns().stream().anyMatch(c -> c.getName().equals(col.getName()));
}
// For test
protected void setTable(HMSExternalTable table) {
- this.table = table;
+ setTable((ExternalTable) table);
+ this.hmsExternalTable = table;
}
- /**
- * Get table row count
- */
- private void getTableStats() throws Exception {
- Map<String, String> params = buildStatsParams(null);
- List<ResultRow> columnResult =
- StatisticsUtil.execStatisticQuery(new StringSubstitutor(params)
- .replace(ANALYZE_TABLE_COUNT_TEMPLATE));
- String rowCount = columnResult.get(0).get(0);
- Env.getCurrentEnv().getAnalysisManager()
- .updateTableStatsStatus(
- new TableStatsMeta(Long.parseLong(rowCount), info, tbl));
- job.rowCountDone(this);
- }
- /**
- * Get column statistics and insert the result to __internal_schema.column_statistics
- */
- protected void getTableColumnStats() throws Exception {
+ @Override
+ protected void getOrdinaryColumnStats() throws Exception {
if (!info.usingSqlForPartitionColumn) {
try {
if (isPartitionColumn()) {
@@ -102,77 +72,17 @@ public class HMSAnalysisTask extends BaseAnalysisTask {
+ "fallback to normal collection",
isPartitionColumn() ? "partition " : "", col.getName(), e);
/* retry using sql way! */
- getOrdinaryColumnStats();
+ super.getOrdinaryColumnStats();
}
} else {
- getOrdinaryColumnStats();
+ super.getOrdinaryColumnStats();
}
}
- private boolean isPartitionColumn() {
- return table.getPartitionColumns().stream().anyMatch(c -> c.getName().equals(col.getName()));
- }
-
- // Get ordinary column stats. Ordinary column means not partition column.
- private void getOrdinaryColumnStats() throws Exception {
- StringBuilder sb = new StringBuilder();
- Map<String, String> params = buildStatsParams("NULL");
- params.put("min", getMinFunction());
- params.put("max", getMaxFunction());
- params.put("dataSizeFunction", getDataSizeFunction(col, false));
- Pair<Double, Long> sampleInfo = getSampleInfo();
- params.put("scaleFactor", String.valueOf(sampleInfo.first));
- StringSubstitutor stringSubstitutor;
- if (tableSample == null) {
- // Do full analyze
- LOG.debug("Will do full collection for column {}", col.getName());
- sb.append(COLLECT_COL_STATISTICS);
- } else {
- // Do sample analyze
- LOG.debug("Will do sample collection for column {}", col.getName());
- boolean limitFlag = false;
- boolean bucketFlag = false;
- // If sample size is too large, use limit to control the sample size.
- if (needLimit(sampleInfo.second, sampleInfo.first)) {
- limitFlag = true;
- long columnSize = 0;
- for (Column column : table.getFullSchema()) {
- columnSize += column.getDataType().getSlotSize();
- }
- double targetRows = (double) sampleInfo.second / columnSize;
- // Estimate the new scaleFactor based on the schema.
- if (targetRows > StatisticsUtil.getHugeTableSampleRows()) {
- params.put("limit", "limit " + StatisticsUtil.getHugeTableSampleRows());
- params.put("scaleFactor",
- String.valueOf(sampleInfo.first * targetRows / StatisticsUtil.getHugeTableSampleRows()));
- }
- }
- // Single distribution column is not fit for DUJ1 estimator, use linear estimator.
- Set<String> distributionColumns = tbl.getDistributionColumnNames();
- if (distributionColumns.size() == 1 && distributionColumns.contains(col.getName().toLowerCase())) {
- bucketFlag = true;
- sb.append(LINEAR_ANALYZE_TEMPLATE);
- params.put("ndvFunction", "ROUND(NDV(`${colName}`) * ${scaleFactor})");
- params.put("rowCount", "ROUND(count(1) * ${scaleFactor})");
- } else {
- sb.append(DUJ1_ANALYZE_TEMPLATE);
- params.put("dataSizeFunction", getDataSizeFunction(col, true));
- params.put("ndvFunction", getNdvFunction("ROUND(SUM(t1.count) * ${scaleFactor})"));
- params.put("rowCount", "ROUND(SUM(t1.count) * ${scaleFactor})");
- }
- LOG.info("Sample for column [{}]. Scale factor [{}], "
- + "limited [{}], is distribute column [{}]",
- col.getName(), params.get("scaleFactor"), limitFlag, bucketFlag);
- }
- stringSubstitutor = new StringSubstitutor(params);
- String sql = stringSubstitutor.replace(sb.toString());
- runQuery(sql);
- }
-
// Collect the partition column stats through HMS metadata.
// Get all the partition values and calculate the stats based on the values.
private void getPartitionColumnStats() throws Exception {
- Set<String> partitionNames = table.getPartitionNames();
+ Set<String> partitionNames = hmsExternalTable.getPartitionNames();
Set<String> ndvPartValues = Sets.newHashSet();
long numNulls = 0;
long dataSize = 0;
@@ -198,8 +108,9 @@ public class HMSAnalysisTask extends BaseAnalysisTask {
}
}
// Estimate the row count. This value is inaccurate if the table stats is empty.
- TableStatsMeta tableStatsStatus = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(table.getId());
- long count = tableStatsStatus == null ? table.estimatedRowCount() : tableStatsStatus.rowCount;
+ TableStatsMeta tableStatsStatus = Env.getCurrentEnv().getAnalysisManager()
+ .findTableStatsStatus(hmsExternalTable.getId());
+ long count = tableStatsStatus == null ? hmsExternalTable.estimatedRowCount() : tableStatsStatus.rowCount;
dataSize = dataSize * count / partitionNames.size();
numNulls = numNulls * count / partitionNames.size();
int ndv = ndvPartValues.size();
@@ -218,8 +129,9 @@ public class HMSAnalysisTask extends BaseAnalysisTask {
// Collect the spark analyzed column stats through HMS metadata.
private void getHmsColumnStats() throws Exception {
- TableStatsMeta tableStatsStatus = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(table.getId());
- long count = tableStatsStatus == null ? table.estimatedRowCount() : tableStatsStatus.rowCount;
+ TableStatsMeta tableStatsStatus = Env.getCurrentEnv().getAnalysisManager()
+ .findTableStatsStatus(hmsExternalTable.getId());
+ long count = tableStatsStatus == null ? hmsExternalTable.estimatedRowCount() : tableStatsStatus.rowCount;
Map<String, String> params = buildStatsParams("NULL");
Map<StatsType, String> statsParams = new HashMap<>();
@@ -229,7 +141,7 @@ public class HMSAnalysisTask extends BaseAnalysisTask {
statsParams.put(StatsType.MAX_VALUE, "max");
statsParams.put(StatsType.AVG_SIZE, "avg_len");
- if (table.fillColumnStatistics(info.colName, statsParams, params)) {
+ if (hmsExternalTable.fillColumnStatistics(info.colName, statsParams, params)) {
throw new AnalysisException("some column stats not available");
}
@@ -283,126 +195,4 @@ public class HMSAnalysisTask extends BaseAnalysisTask {
}
return value.compareTo(currentMax) > 0 ? value : currentMax;
}
-
- private Map<String, String> buildStatsParams(String partId) {
- Map<String, String> commonParams = new HashMap<>();
- String id = StatisticsUtil.constructId(tbl.getId(), -1);
- if (partId == null) {
- commonParams.put("partId", "NULL");
- } else {
- id = StatisticsUtil.constructId(id, partId);
- commonParams.put("partId", "\'" + partId + "\'");
- }
- commonParams.put("internalDB", FeConstants.INTERNAL_DB_NAME);
- commonParams.put("columnStatTbl", StatisticConstants.STATISTIC_TBL_NAME);
- commonParams.put("id", id);
- commonParams.put("catalogId", String.valueOf(catalog.getId()));
- commonParams.put("dbId", String.valueOf(db.getId()));
- commonParams.put("tblId", String.valueOf(tbl.getId()));
- commonParams.put("indexId", "-1");
- commonParams.put("idxId", "-1");
- commonParams.put("colName", info.colName);
- commonParams.put("colId", info.colName);
- commonParams.put("catalogName", catalog.getName());
- commonParams.put("dbName", db.getFullName());
- commonParams.put("tblName", tbl.getName());
- commonParams.put("sampleHints", getSampleHint());
- commonParams.put("limit", "");
- commonParams.put("scaleFactor", "1");
- if (col != null) {
- commonParams.put("type", col.getType().toString());
- }
- commonParams.put("lastAnalyzeTimeInMs", String.valueOf(System.currentTimeMillis()));
- return commonParams;
- }
-
- protected String getSampleHint() {
- if (tableSample == null) {
- return "";
- }
- if (tableSample.isPercent()) {
- return String.format("TABLESAMPLE(%d PERCENT)", tableSample.getSampleValue());
- } else {
- return String.format("TABLESAMPLE(%d ROWS)", tableSample.getSampleValue());
- }
- }
-
- /**
- * Get the pair of sample scale factor and the file size going to sample.
- * While analyzing, the result of count, null count and data size need to
- * multiply this scale factor to get more accurate result.
- * @return Pair of sample scale factor and the file size going to sample.
- */
- protected Pair<Double, Long> getSampleInfo() {
- if (tableSample == null) {
- return Pair.of(1.0, 0L);
- }
- long target;
- // Get list of all files' size in this HMS table.
- List<Long> chunkSizes = table.getChunkSizes();
- Collections.shuffle(chunkSizes, new Random(tableSample.getSeek()));
- long total = 0;
- // Calculate the total size of this HMS table.
- for (long size : chunkSizes) {
- total += size;
- }
- if (total == 0) {
- return Pair.of(1.0, 0L);
- }
- // Calculate the sample target size for percent and rows sample.
- if (tableSample.isPercent()) {
- target = total * tableSample.getSampleValue() / 100;
- } else {
- int columnSize = 0;
- for (Column column : table.getFullSchema()) {
- columnSize += column.getDataType().getSlotSize();
- }
- target = columnSize * tableSample.getSampleValue();
- }
- // Calculate the actual sample size (cumulate).
- long cumulate = 0;
- for (long size : chunkSizes) {
- cumulate += size;
- if (cumulate >= target) {
- break;
- }
- }
- return Pair.of(Math.max(((double) total) / cumulate, 1), cumulate);
- }
-
- @Override
- protected void afterExecution() {
- // Table level task doesn't need to sync any value to sync stats, it stores the value in metadata.
- // Partition only task doesn't need to refresh cached.
- if (isTableLevelTask || isPartitionOnly) {
- return;
- }
- Env.getCurrentEnv().getStatisticsCache().syncLoadColStats(tbl.getId(), -1, col.getName());
- }
-
- /**
- * If the size to sample is larger than LIMIT_SIZE (1GB)
- * and is much larger (1.2*) than the size user want to sample,
- * use limit to control the total sample size.
- * @param sizeToRead The file size to sample.
- * @param factor sizeToRead * factor = Table total size.
- * @return True if need to limit.
- */
- protected boolean needLimit(long sizeToRead, double factor) {
- long total = (long) (sizeToRead * factor);
- long target;
- if (tableSample.isPercent()) {
- target = total * tableSample.getSampleValue() / 100;
- } else {
- int columnSize = 0;
- for (Column column : table.getFullSchema()) {
- columnSize += column.getDataType().getSlotSize();
- }
- target = columnSize * tableSample.getSampleValue();
- }
- if (sizeToRead > LIMIT_SIZE && sizeToRead > target * LIMIT_FACTOR) {
- return true;
- }
- return false;
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index 5c8aec3fbf6..6176ec13bd6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -84,10 +84,12 @@ import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Table;
import org.apache.iceberg.TableScan;
+import org.apache.iceberg.io.CloseableIterable;
import org.apache.iceberg.types.Types;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
@@ -735,8 +737,12 @@ public class StatisticsUtil {
columnStatisticBuilder.setDataSize(0);
columnStatisticBuilder.setAvgSizeByte(0);
columnStatisticBuilder.setNumNulls(0);
- for (FileScanTask task : tableScan.planFiles()) {
- processDataFile(task.file(), task.spec(), colName, columnStatisticBuilder);
+ try (CloseableIterable<FileScanTask> fileScanTasks = tableScan.planFiles()) {
+ for (FileScanTask task : fileScanTasks) {
+ processDataFile(task.file(), task.spec(), colName, columnStatisticBuilder);
+ }
+ } catch (IOException e) {
+ LOG.warn("Error to close FileScanTask.", e);
}
if (columnStatisticBuilder.getCount() > 0) {
columnStatisticBuilder.setAvgSizeByte(columnStatisticBuilder.getDataSize()
diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/HMSAnalysisTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/HMSAnalysisTaskTest.java
index fb0a3b3c2ca..e1016864525 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/statistics/HMSAnalysisTaskTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/HMSAnalysisTaskTest.java
@@ -252,7 +252,7 @@ public class HMSAnalysisTaskTest {
analysisInfoBuilder.setUsingSqlForPartitionColumn(true);
task.info = analysisInfoBuilder.build();
- task.getTableColumnStats();
+ task.getOrdinaryColumnStats();
}
@@ -309,6 +309,6 @@ public class HMSAnalysisTaskTest {
analysisInfoBuilder.setUsingSqlForPartitionColumn(false);
task.info = analysisInfoBuilder.build();
- task.getTableColumnStats();
+ task.getOrdinaryColumnStats();
}
}
diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy
index 39c41f7c11a..6d17bd032fd 100644
--- a/regression-test/conf/regression-conf.groovy
+++ b/regression-test/conf/regression-conf.groovy
@@ -191,6 +191,7 @@ extArrowFlightSqlPassword= ""
// iceberg rest catalog config
iceberg_rest_uri_port=18181
+iceberg_minio_port=19001
// If the failure suite num exceeds this config
// all following suite will be skipped to fast quit the run.
diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_statistics.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_statistics.out
new file mode 100644
index 00000000000..c094d171479
--- /dev/null
+++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_statistics.out
@@ -0,0 +1,39 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !s1 --
+city 1000 4 0 Beijing Shanghai 6973
+col_binary 1000 867 0 0 1111101100100001001 15356
+col_boolean 1000 2 0 0 1 1000
+col_byte 1000 251 0 -128 127 4000
+col_char 1000 963 0 ! zy@notj#fkedb($ 9348
+col_date 1000 3 0 1969-09-21 2969-02-03 4000
+col_decimal 1000 1006 0 4.028284 9999.512216 8000
+col_double 1000 990 0 0.005217837593576302 9.996285421163707 8000
+col_float 1000 995 0 0.013126845 9.99709 4000
+col_integer 1000 999 0 -21468189 2108484 4000
+col_long 1000 996 0 -92193877774291102 92127291905311066 8000
+col_short 1000 985 0 -32554 32525 4000
+col_string 1000 992 0 0 zx70Jyeb6TfQ1YUaIGC 10714
+col_timestamp 1000 4 0 1970-01-01 08:00:01.000001 1970-01-04 08:00:01.000001 8000
+col_timestamp_ntz 1000 4 0 2017-12-01 10:12:55.038194 2017-12-04 10:12:55.038194 8000
+col_varchar 1000 988 0 0 zvnZ6bBxh 10764
+id 1000 1001 0 -99567408 99854631 8000
+
+-- !s2 --
+city 1000 4 0 Beijing Shanghai 6973
+col_binary 1000 867 0 0 1111101100100001001 15356
+col_boolean 1000 2 0 0 1 1000
+col_byte 1000 251 0 -128 127 4000
+col_char 1000 973 0 ! zy@notj#fkedb($ 9324
+col_date 1000 3 0 1969-09-21 2969-02-03 4000
+col_decimal 1000 1006 0 4.028284 9999.512216 8000
+col_double 1000 990 0 0.005217837593576302 9.996285421163707 8000
+col_float 1000 995 0 0.013126845 9.99709 4000
+col_integer 1000 999 0 -21468189 2108484 4000
+col_long 1000 996 0 -92193877774291102 92127291905311066 8000
+col_short 1000 985 0 -32554 32525 4000
+col_string 1000 992 0 0 zx70Jyeb6TfQ1YUaIGC 10714
+col_timestamp 1000 4 0 1970-01-01 08:00:01.000001 1970-01-04 08:00:01.000001 8000
+col_timestamp_ntz 1000 4 0 2017-12-01 10:12:55.038194 2017-12-04 10:12:55.038194 8000
+col_varchar 1000 988 0 0 zvnZ6bBxh 10764
+id 1000 1001 0 -99567408 99854631 8000
+
diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_statistics.out b/regression-test/data/external_table_p0/paimon/test_paimon_statistics.out
new file mode 100644
index 00000000000..0f9f20d4782
--- /dev/null
+++ b/regression-test/data/external_table_p0/paimon/test_paimon_statistics.out
@@ -0,0 +1,21 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !s1 --
+c1 2 2 0 1 10 2
+c10 2 2 0 10.1 100.1 16
+c11 2 2 0 11.10 110.10 16
+c12 2 2 0 2020-02-02 2020-03-02 8
+c13 2 2 0 130str 13str 11
+c14 2 2 0 140varchar 14varchar 19
+c15 2 2 0 a b 2
+c16 2 2 0 0 1 2
+c17 2 2 0 aaaa bbbb 8
+c18 2 2 0 2023-08-13 09:32:38.530000 2023-08-14 08:32:52.821000 16
+c2 2 2 0 2 20 2
+c3 2 2 0 3 30 4
+c4 2 2 0 4 40 4
+c5 2 2 0 5 50 8
+c6 2 2 0 6 60 8
+c7 2 2 0 7 70 16
+c8 2 2 0 8 80 16
+c9 2 2 0 9.1 90.1 8
+
diff --git a/regression-test/data/external_table_p2/hive/test_hive_hudi_statistics.out b/regression-test/data/external_table_p2/hive/test_hive_hudi_statistics.out
new file mode 100644
index 00000000000..66a36a81afc
--- /dev/null
+++ b/regression-test/data/external_table_p2/hive/test_hive_hudi_statistics.out
@@ -0,0 +1,16 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !s1 --
+_hoodie_commit_seqno 4 4 0 20230605145009209_0_1 20230801201335031_1_1 84
+_hoodie_commit_time 4 3 0 20230605145009209 20230801201335031 68
+_hoodie_file_name 4 4 0 65ffc5d9-397a-456e-a735-30f3ad37466f-0 e33d645c-6e2f-41f3-b8d6-f658771bd460-0_1-83-220_20230605145403388.parquet 221
+_hoodie_partition_path 4 3 0 partitionId=2011-11-11/versionId=v_1 partitionId=2021-02-01/versionId=v_4 144
+_hoodie_record_key 4 3 0 rowId:row_1 rowId:row_4 44
+inttolong 4 2 0 0 1 16
+longtoint 4 3 0 1000000 1000004 32
+name 4 3 0 ashin john 15
+partitionid 4 3 0 2011-11-11 2021-02-01 40
+precomb 4 3 0 0 4 32
+rowid 4 3 0 row_1 row_4 20
+tobedeletedstr 4 3 0 toBeDel0 toBeDel4 32
+versionid 4 3 0 v_0 v_4 12
+
diff --git a/regression-test/pipeline/p0/conf/regression-conf.groovy b/regression-test/pipeline/p0/conf/regression-conf.groovy
index be70ab0f730..b99e21c4e7c 100644
--- a/regression-test/pipeline/p0/conf/regression-conf.groovy
+++ b/regression-test/pipeline/p0/conf/regression-conf.groovy
@@ -96,6 +96,7 @@ kafka_port=19193
// iceberg test config
iceberg_rest_uri_port=18181
+iceberg_minio_port=19001
enableEsTest=false
es_6_port=19200
diff --git a/regression-test/pipeline/tpch/tpch-sf100/conf/regression-conf.groovy b/regression-test/pipeline/tpch/tpch-sf100/conf/regression-conf.groovy
index 364a7103fe8..5234ccc4241 100644
--- a/regression-test/pipeline/tpch/tpch-sf100/conf/regression-conf.groovy
+++ b/regression-test/pipeline/tpch/tpch-sf100/conf/regression-conf.groovy
@@ -94,6 +94,7 @@ kafka_port=19193
// iceberg test config
iceberg_rest_uri_port=18181
+iceberg_minio_port=19001
enableEsTest=false
es_6_port=19200
diff --git a/regression-test/plugins/plugins_get_ids_from_proc.groovy b/regression-test/plugins/plugins_get_ids_from_proc.groovy
new file mode 100644
index 00000000000..74a4d4d2010
--- /dev/null
+++ b/regression-test/plugins/plugins_get_ids_from_proc.groovy
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.doris.regression.suite.Suite
+
+Suite.metaClass.get_catalog_id = {String catalog_name /* param */ ->
+ String catalog_id;
+ def catalogs = sql """show proc '/catalogs'"""
+ for (catalog in catalogs) {
+ if (catalog[1].equals(catalog_name)) {
+ catalog_id = catalog[0]
+ break
+ }
+ }
+ log.info("get catalogid: " + catalog_id)
+ return catalog_id
+}
+
+
+Suite.metaClass.get_database_id = {String catalog_name, String db_name /* param */ ->
+ String database_id;
+ def catalog_id = get_catalog_id(catalog_name)
+ def dbs = sql """show proc '/catalogs/${catalog_id}'"""
+ for (db in dbs) {
+ if (db[1].equals(db_name)) {
+ database_id = db[0]
+ break
+ }
+ }
+ log.info("get database_id: " + database_id)
+ return database_id
+}
+
+
+Suite.metaClass.get_table_id = {String catalog_name, String db_name, String tb_name /* param */ ->
+ String table_id;
+ def catalog_id = get_catalog_id(catalog_name)
+ def database_id = get_database_id(catalog_name, db_name)
+ def tbs = sql """show proc '/catalogs/${catalog_id}/${database_id}'"""
+ for (tb in tbs) {
+ if (tb[1].equals(tb_name)) {
+ table_id = tb[0]
+ break
+ }
+ }
+ log.info("get table_id: " + table_id)
+ return table_id
+}
diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_statistics.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_statistics.groovy
new file mode 100644
index 00000000000..24b27eb70b4
--- /dev/null
+++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_statistics.groovy
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_iceberg_statistics", "p0,external,doris,external_docker,external_docker_doris") {
+ String enabled = context.config.otherConfigs.get("enableIcebergTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ try {
+ String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port")
+ String minio_port = context.config.otherConfigs.get("iceberg_minio_port")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String catalog_name = "test_iceberg_rest_catalog"
+ String db_name = "format_v2"
+
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """CREATE CATALOG ${catalog_name} PROPERTIES (
+ 'type'='iceberg',
+ 'iceberg.catalog.type'='rest',
+ 'uri' = 'http://${externalEnvIp}:${rest_port}',
+ "s3.access_key" = "admin",
+ "s3.secret_key" = "password",
+ "s3.endpoint" = "http://${externalEnvIp}:${minio_port}",
+ "s3.region" = "us-east-1"
+ );"""
+
+ def table_id_mor = get_table_id(catalog_name, db_name, "sample_mor_parquet")
+ def table_id_cow = get_table_id(catalog_name, db_name, "sample_cow_parquet")
+
+ // analyze
+ sql """use `${catalog_name}`.`${db_name}`"""
+ sql """analyze table sample_mor_parquet with sync"""
+ sql """analyze table sample_cow_parquet with sync"""
+
+ // select
+ def s1 = """select col_id,count,ndv,null_count,min,max,data_size_in_bytes from internal.__internal_schema.column_statistics where tbl_id = ${table_id_mor} order by id;"""
+ def s2 = """select col_id,count,ndv,null_count,min,max,data_size_in_bytes from internal.__internal_schema.column_statistics where tbl_id = ${table_id_cow} order by id;"""
+
+ qt_s1 s1
+ qt_s2 s2
+ } finally {
+ }
+ }
+}
+
diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
index 87ea14ad2fd..ce8c9b5e849 100644
--- a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
+++ b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
@@ -52,6 +52,9 @@ suite("test_paimon_catalog", "p0,external,doris,external_docker,external_docker_
);
"""
+ sql """drop catalog ${file_ctl_name}""";
+ sql """drop catalog ${hms_ctl_name}""";
+
String enabled = context.config.otherConfigs.get("enablePaimonTest")
if (enabled != null && enabled.equalsIgnoreCase("true")) {
def all = """select * from all_table order by c1;"""
diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_statistics.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_statistics.groovy
new file mode 100644
index 00000000000..c75e7b797d9
--- /dev/null
+++ b/regression-test/suites/external_table_p0/paimon/test_paimon_statistics.groovy
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_paimon_statistics", "p0,external,doris,external_docker,external_docker_doris") {
+ String enabled = context.config.otherConfigs.get("enablePaimonTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ try {
+ String hdfs_port = context.config.otherConfigs.get("hdfs_port")
+ String catalog_name = "paimon1"
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """create catalog if not exists ${catalog_name} properties (
+ "type" = "paimon",
+ "paimon.catalog.type"="filesystem",
+ "warehouse" = "hdfs://${externalEnvIp}:${hdfs_port}/user/doris/paimon1"
+ );"""
+
+ def table_id = get_table_id(catalog_name, "db1", "all_table")
+
+ // analyze
+ sql """use `${catalog_name}`.`db1`"""
+ sql """analyze table all_table with sync"""
+
+ // select
+ def s1 = """select col_id,count,ndv,null_count,min,max,data_size_in_bytes from internal.__internal_schema.column_statistics where tbl_id = ${table_id} order by id;"""
+
+ qt_s1 s1
+ } finally {
+ }
+ }
+}
+
diff --git a/regression-test/suites/external_table_p0/test_catalog_ddl.groovy b/regression-test/suites/external_table_p0/test_catalog_ddl.groovy
index b236567e8bd..a9a67f51853 100644
--- a/regression-test/suites/external_table_p0/test_catalog_ddl.groovy
+++ b/regression-test/suites/external_table_p0/test_catalog_ddl.groovy
@@ -44,4 +44,6 @@ suite("test_catalog_ddl", "p0,external,external_docker") {
result = sql """show create catalog ${catalog1};"""
assertEquals(result.size(), 1)
assertTrue(result[0][1].contains("COMMENT \"alter_comment\""))
+
+ sql """drop catalog ${catalog1}"""
}
diff --git a/regression-test/suites/external_table_p2/hive/test_hive_hudi_statistics.groovy b/regression-test/suites/external_table_p2/hive/test_hive_hudi_statistics.groovy
new file mode 100644
index 00000000000..55e5037de45
--- /dev/null
+++ b/regression-test/suites/external_table_p2/hive/test_hive_hudi_statistics.groovy
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_hive_hudi_statistics", "p2,external,hive,hudi") {
+ String enabled = context.config.otherConfigs.get("enableExternalHiveTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String extHiveHmsHost = context.config.otherConfigs.get("extHiveHmsHost")
+ String extHiveHmsPort = context.config.otherConfigs.get("extHiveHmsPort")
+ String catalog_name = "test_hive_hudi_statistics"
+ String db_name = "hudi_catalog"
+
+ sql """drop catalog if exists ${catalog_name};"""
+ sql """
+ create catalog if not exists ${catalog_name} properties (
+ 'hadoop.username'='hadoop',
+ 'type'='hms',
+ 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}'
+ );
+ """
+
+ def table_id = get_table_id(catalog_name, db_name, "partitioned_mor_rt")
+
+ // analyze
+ sql """use `${catalog_name}`.`${db_name}`"""
+ sql """analyze table partitioned_mor_rt with sync"""
+
+ // select
+ def s1 = """select col_id,count,ndv,null_count,min,max,data_size_in_bytes from internal.__internal_schema.column_statistics where tbl_id = ${table_id} order by id;"""
+
+ qt_s1 s1
+
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 03/22: [fix](fe ut) fix unstable TabletRepairAndBalanceTest (#30041)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 2ca3c821351d6c18d63e6b90d0bb4330c3145d00
Author: yujun <yu...@gmail.com>
AuthorDate: Wed Jan 17 14:25:47 2024 +0800
[fix](fe ut) fix unstable TabletRepairAndBalanceTest (#30041)
---
fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java | 4 ++++
.../test/java/org/apache/doris/clone/TabletRepairAndBalanceTest.java | 4 +++-
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
index e579c6b9a48..3ec3bbeef51 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
@@ -692,6 +692,10 @@ public class Tablet extends MetaObject implements Writable {
* LOW: delay Config.tablet_repair_delay_factor_second * 3;
*/
public boolean readyToBeRepaired(SystemInfoService infoService, TabletSchedCtx.Priority priority) {
+ if (FeConstants.runningUnitTest) {
+ return true;
+ }
+
if (priority == Priority.VERY_HIGH) {
return true;
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/clone/TabletRepairAndBalanceTest.java b/fe/fe-core/src/test/java/org/apache/doris/clone/TabletRepairAndBalanceTest.java
index 9f9fd8a0942..979943b1609 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/clone/TabletRepairAndBalanceTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/clone/TabletRepairAndBalanceTest.java
@@ -106,9 +106,11 @@ public class TabletRepairAndBalanceTest {
FeConstants.runningUnitTest = true;
System.out.println(runningDir);
FeConstants.runningUnitTest = true;
- Config.tablet_checker_interval_ms = 1000;
+ Config.tablet_checker_interval_ms = 100;
Config.tablet_repair_delay_factor_second = 1;
Config.colocate_group_relocate_delay_second = 1;
+ Config.schedule_slot_num_per_hdd_path = 1000;
+ Config.schedule_slot_num_per_ssd_path = 1000;
Config.disable_balance = true;
// 5 backends:
// 127.0.0.1
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 20/22: [feature](statistics, metadata)Meta data place holder for statistics (#29867)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit cbac86324adacbe80e211cb8341f9e8f0a5c67fc
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Wed Jan 17 22:41:44 2024 +0800
[feature](statistics, metadata)Meta data place holder for statistics (#29867)
Meta data place holder for statistics in version 2.1.x. Users could upgrade to this version, but doesn't support rollback.
After this change, statistics related functions doesn't need to change meta data any more in the 2.1 series.
---
.../org/apache/doris/common/FeMetaVersion.java | 4 +-
.../java/org/apache/doris/persist/EditLog.java | 22 ++++++--
.../org/apache/doris/persist/OperationType.java | 6 ++
.../apache/doris/statistics/AnalysisJobInfo.java | 54 ++++++++++++++++++
.../apache/doris/statistics/AnalysisManager.java | 46 +++------------
.../apache/doris/statistics/AnalysisTaskInfo.java | 58 +++++++++++++++++++
.../doris/statistics/NewPartitionLoadedEvent.java | 57 +++++++++++++++++++
.../apache/doris/statistics/UpdateRowsEvent.java | 61 ++++++++++++++++++++
.../apache/doris/statistics/util/SimpleQueue.java | 65 ----------------------
.../doris/statistics/AnalysisManagerTest.java | 24 --------
10 files changed, 264 insertions(+), 133 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java
index 387ce91e2c6..9f5474fa01c 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java
@@ -74,9 +74,11 @@ public final class FeMetaVersion {
public static final int VERSION_126 = 126;
// For constraints
public static final int VERSION_127 = 127;
+ // For statistics. Update rows, new partition loaded, AnalysisJobInfo and AnalysisTaskInfo
+ public static final int VERSION_128 = 128;
// note: when increment meta version, should assign the latest version to VERSION_CURRENT
- public static final int VERSION_CURRENT = VERSION_127;
+ public static final int VERSION_CURRENT = VERSION_128;
// all logs meta version should >= the minimum version, so that we could remove many if clause, for example
// if (FE_METAVERSION < VERSION_94) ...
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
index 5b2ff25f54e..8c2424d4837 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
@@ -81,7 +81,9 @@ import org.apache.doris.policy.StoragePolicy;
import org.apache.doris.resource.workloadgroup.WorkloadGroup;
import org.apache.doris.resource.workloadschedpolicy.WorkloadSchedPolicy;
import org.apache.doris.statistics.AnalysisInfo;
+import org.apache.doris.statistics.AnalysisJobInfo;
import org.apache.doris.statistics.AnalysisManager;
+import org.apache.doris.statistics.AnalysisTaskInfo;
import org.apache.doris.statistics.TableStatsMeta;
import org.apache.doris.system.Backend;
import org.apache.doris.system.Frontend;
@@ -1080,6 +1082,10 @@ public class EditLog {
break;
}
case OperationType.OP_CREATE_ANALYSIS_JOB: {
+ if (journal.getData() instanceof AnalysisJobInfo) {
+ // For rollback compatible.
+ break;
+ }
AnalysisInfo info = (AnalysisInfo) journal.getData();
if (AnalysisManager.needAbandon(info)) {
break;
@@ -1088,6 +1094,10 @@ public class EditLog {
break;
}
case OperationType.OP_CREATE_ANALYSIS_TASK: {
+ if (journal.getData() instanceof AnalysisTaskInfo) {
+ // For rollback compatible.
+ break;
+ }
AnalysisInfo info = (AnalysisInfo) journal.getData();
if (AnalysisManager.needAbandon(info)) {
break;
@@ -1131,7 +1141,7 @@ public class EditLog {
break;
}
case OperationType.OP_PERSIST_AUTO_JOB: {
- env.getAnalysisManager().replayPersistSysJob((AnalysisInfo) journal.getData());
+ // Do nothing
break;
}
case OperationType.OP_DELETE_TABLE_STATS: {
@@ -1155,6 +1165,12 @@ public class EditLog {
env.getBackupHandler().getRepoMgr().alterRepo(repository, true);
break;
}
+ case OperationType.OP_LOG_UPDATE_ROWS:
+ case OperationType.OP_LOG_NEW_PARTITION_LOADED:
+ case OperationType.OP_LOG_ALTER_COLUMN_STATS: {
+ // TODO: implement this while statistics finished related work.
+ break;
+ }
default: {
IOException e = new IOException();
LOG.error("UNKNOWN Operation Type {}", opCode, e);
@@ -1968,10 +1984,6 @@ public class EditLog {
logEdit(OperationType.OP_UPDATE_TABLE_STATS, tableStats);
}
- public void logAutoJob(AnalysisInfo analysisInfo) {
- logEdit(OperationType.OP_PERSIST_AUTO_JOB, analysisInfo);
- }
-
public void logDeleteTableStats(TableStatsDeletionLog log) {
logEdit(OperationType.OP_DELETE_TABLE_STATS, log);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java
index 42312297b91..0945dc0f151 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java
@@ -345,6 +345,7 @@ public class OperationType {
public static final short OP_UPDATE_TABLE_STATS = 455;
+ @Deprecated
public static final short OP_PERSIST_AUTO_JOB = 456;
public static final short OP_DELETE_TABLE_STATS = 457;
@@ -357,6 +358,11 @@ public class OperationType {
public static final short OP_INSERT_OVERWRITE = 461;
+ public static final short OP_LOG_UPDATE_ROWS = 462;
+
+ public static final short OP_LOG_NEW_PARTITION_LOADED = 463;
+
+ public static final short OP_LOG_ALTER_COLUMN_STATS = 464;
/**
* Get opcode name by op code.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJobInfo.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJobInfo.java
new file mode 100644
index 00000000000..57ae461fe13
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJobInfo.java
@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.statistics;
+
+import org.apache.doris.common.io.Text;
+import org.apache.doris.common.io.Writable;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import com.google.gson.annotations.SerializedName;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+public class AnalysisJobInfo implements Writable {
+
+ private static final Logger LOG = LogManager.getLogger(AnalysisJobInfo.class);
+
+ @SerializedName("jobId")
+ public final long jobId;
+
+ public AnalysisJobInfo(long jobId) {
+ this.jobId = jobId;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ String json = GsonUtils.GSON.toJson(this);
+ Text.writeString(out, json);
+ }
+
+ public static AnalysisJobInfo read(DataInput dataInput) throws IOException {
+ String json = Text.readString(dataInput);
+ AnalysisJobInfo analysisJobInfo = GsonUtils.GSON.fromJson(json, AnalysisJobInfo.class);
+ return analysisJobInfo;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index 00e858c1536..2e12b443390 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -39,6 +39,7 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.ThreadPoolManager.BlockedPolicy;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
@@ -56,7 +57,6 @@ import org.apache.doris.statistics.AnalysisInfo.AnalysisType;
import org.apache.doris.statistics.AnalysisInfo.JobType;
import org.apache.doris.statistics.AnalysisInfo.ScheduleType;
import org.apache.doris.statistics.util.DBObjects;
-import org.apache.doris.statistics.util.SimpleQueue;
import org.apache.doris.statistics.util.StatisticsUtil;
import com.google.common.annotations.VisibleForTesting;
@@ -122,9 +122,6 @@ public class AnalysisManager implements Writable {
private final Map<Long, AnalysisJob> idToAnalysisJob = new ConcurrentHashMap<>();
- // To be deprecated, keep it for meta compatibility now, will remove later.
- protected SimpleQueue<AnalysisInfo> autoJobs = createSimpleQueue(null, this);
-
private final String progressDisplayTemplate = "%d Finished | %d Failed | %d In Progress | %d Total";
public AnalysisManager() {
@@ -862,13 +859,16 @@ public class AnalysisManager implements Writable {
readAnalysisInfo(in, analysisManager.analysisJobInfoMap, true);
readAnalysisInfo(in, analysisManager.analysisTaskInfoMap, false);
readIdToTblStats(in, analysisManager.idToTblStats);
- readAutoJobs(in, analysisManager);
+ if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_128) {
+ readAutoJobs(in, analysisManager);
+ }
return analysisManager;
}
private static void readAnalysisInfo(DataInput in, Map<Long, AnalysisInfo> map, boolean job) throws IOException {
int size = in.readInt();
for (int i = 0; i < size; i++) {
+ // AnalysisInfo is compatible with AnalysisJobInfo and AnalysisTaskInfo.
AnalysisInfo analysisInfo = AnalysisInfo.read(in);
// Unfinished manual once job/tasks doesn't need to keep in memory anymore.
if (needAbandon(analysisInfo)) {
@@ -884,6 +884,9 @@ public class AnalysisManager implements Writable {
if (analysisInfo == null) {
return true;
}
+ if (analysisInfo.scheduleType == null || analysisInfo.scheduleType == null || analysisInfo.jobType == null) {
+ return true;
+ }
if ((AnalysisState.PENDING.equals(analysisInfo.state) || AnalysisState.RUNNING.equals(analysisInfo.state))
&& ScheduleType.ONCE.equals(analysisInfo.scheduleType)
&& JobType.MANUAL.equals(analysisInfo.jobType)) {
@@ -904,7 +907,6 @@ public class AnalysisManager implements Writable {
private static void readAutoJobs(DataInput in, AnalysisManager analysisManager) throws IOException {
Type type = new TypeToken<LinkedList<AnalysisInfo>>() {}.getType();
GsonUtils.GSON.fromJson(Text.readString(in), type);
- analysisManager.autoJobs = analysisManager.createSimpleQueue(null, null);
}
@Override
@@ -912,7 +914,6 @@ public class AnalysisManager implements Writable {
writeJobInfo(out, analysisJobInfoMap);
writeJobInfo(out, analysisTaskInfoMap);
writeTableStats(out);
- writeAutoJobsStatus(out);
}
private void writeJobInfo(DataOutput out, Map<Long, AnalysisInfo> infoMap) throws IOException {
@@ -929,12 +930,6 @@ public class AnalysisManager implements Writable {
}
}
- private void writeAutoJobsStatus(DataOutput output) throws IOException {
- Type type = new TypeToken<LinkedList<AnalysisInfo>>() {}.getType();
- String autoJobs = GsonUtils.GSON.toJson(this.autoJobs, type);
- Text.writeString(output, autoJobs);
- }
-
// For unit test use only.
public void addToJobIdTasksMap(long jobId, Map<Long, BaseAnalysisTask> tasks) {
analysisJobIdToTaskMap.put(jobId, tasks);
@@ -980,31 +975,6 @@ public class AnalysisManager implements Writable {
analysisJobIdToTaskMap.put(jobInfo.jobId, taskInfos);
}
- protected void logAutoJob(AnalysisInfo autoJob) {
- Env.getCurrentEnv().getEditLog().logAutoJob(autoJob);
- }
-
- public void replayPersistSysJob(AnalysisInfo analysisInfo) {
- autoJobs.offer(analysisInfo);
- }
-
- protected SimpleQueue<AnalysisInfo> createSimpleQueue(Collection<AnalysisInfo> collection,
- AnalysisManager analysisManager) {
- return new SimpleQueue<>(Config.analyze_record_limit,
- a -> {
- // FE is not ready when replaying log and operations triggered by replaying
- // shouldn't be logged again.
- if (Env.getCurrentEnv().isReady() && Env.getCurrentEnv().isMaster() && !Env.isCheckpointThread()) {
- analysisManager.logAutoJob(a);
- }
- return null;
- },
- a -> {
- // DO NOTHING
- return null;
- }, collection);
- }
-
// Remove col stats status from TableStats if failed load some col stats after analyze corresponding column so that
// we could make sure it would be analyzed again soon if user or system submit job for that column again.
public void removeColStatsStatus(long tblId, String colName) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskInfo.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskInfo.java
new file mode 100644
index 00000000000..fb7a5dcbd58
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskInfo.java
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.statistics;
+
+import org.apache.doris.common.io.Text;
+import org.apache.doris.common.io.Writable;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import com.google.gson.annotations.SerializedName;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+public class AnalysisTaskInfo implements Writable {
+
+ private static final Logger LOG = LogManager.getLogger(AnalysisTaskInfo.class);
+
+ @SerializedName("jobId")
+ public final long jobId;
+
+ @SerializedName("taskId")
+ public final long taskId;
+
+ public AnalysisTaskInfo(long jobId, long taskId) {
+ this.jobId = jobId;
+ this.taskId = taskId;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ String json = GsonUtils.GSON.toJson(this);
+ Text.writeString(out, json);
+ }
+
+ public static AnalysisTaskInfo read(DataInput dataInput) throws IOException {
+ String json = Text.readString(dataInput);
+ AnalysisTaskInfo analysisTaskInfo = GsonUtils.GSON.fromJson(json, AnalysisTaskInfo.class);
+ return analysisTaskInfo;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/NewPartitionLoadedEvent.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/NewPartitionLoadedEvent.java
new file mode 100644
index 00000000000..d09cb2df6c4
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/NewPartitionLoadedEvent.java
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.statistics;
+
+import org.apache.doris.common.io.Text;
+import org.apache.doris.common.io.Writable;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.annotations.SerializedName;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class NewPartitionLoadedEvent implements Writable {
+
+ @SerializedName("partitionIdToTableId")
+ public final Map<Long, Long> partitionIdToTableId = new HashMap<>();
+
+ @VisibleForTesting
+ public NewPartitionLoadedEvent() {}
+
+ // No need to be thread safe, only publish thread will call this.
+ public void addPartition(long tableId, long partitionId) {
+ partitionIdToTableId.put(tableId, partitionId);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ String json = GsonUtils.GSON.toJson(this);
+ Text.writeString(out, json);
+ }
+
+ public static NewPartitionLoadedEvent read(DataInput dataInput) throws IOException {
+ String json = Text.readString(dataInput);
+ NewPartitionLoadedEvent newPartitionLoadedEvent = GsonUtils.GSON.fromJson(json, NewPartitionLoadedEvent.class);
+ return newPartitionLoadedEvent;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/UpdateRowsEvent.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/UpdateRowsEvent.java
new file mode 100644
index 00000000000..04f185c8b73
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/UpdateRowsEvent.java
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.statistics;
+
+import org.apache.doris.common.io.Text;
+import org.apache.doris.common.io.Writable;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.annotations.SerializedName;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class UpdateRowsEvent implements Writable {
+
+ @SerializedName("tableIdToUpdateRows")
+ public final Map<Long, Long> tableIdToUpdateRows = new HashMap<>();
+
+ @VisibleForTesting
+ public UpdateRowsEvent() {}
+
+ // No need to be thread safe, only publish thread will call this.
+ public void addUpdateRows(long tableId, long rows) {
+ if (tableIdToUpdateRows.containsKey(tableId)) {
+ tableIdToUpdateRows.put(tableId, tableIdToUpdateRows.get(tableId) + rows);
+ } else {
+ tableIdToUpdateRows.put(tableId, rows);
+ }
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ String json = GsonUtils.GSON.toJson(this);
+ Text.writeString(out, json);
+ }
+
+ public static UpdateRowsEvent read(DataInput dataInput) throws IOException {
+ String json = Text.readString(dataInput);
+ UpdateRowsEvent updateRowsEvent = GsonUtils.GSON.fromJson(json, UpdateRowsEvent.class);
+ return updateRowsEvent;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/SimpleQueue.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/SimpleQueue.java
deleted file mode 100644
index 5740c4e3088..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/SimpleQueue.java
+++ /dev/null
@@ -1,65 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.statistics.util;
-
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.function.Function;
-
-// Any operation on this structure should be thread-safe
-public class SimpleQueue<T> extends LinkedList<T> {
-
- private final long limit;
-
- private final Function<T, Void> offerFunc;
-
- private final Function<T, Void> evictFunc;
-
-
- public SimpleQueue(long limit, Function<T, Void> offerFunc, Function<T, Void> evictFunc) {
- this.limit = limit;
- this.offerFunc = offerFunc;
- this.evictFunc = evictFunc;
- }
-
- @Override
- public synchronized boolean offer(T analysisInfo) {
- while (size() >= limit) {
- remove();
- }
- super.offer(analysisInfo);
- offerFunc.apply(analysisInfo);
- return true;
- }
-
- @Override
- public synchronized T remove() {
- T analysisInfo = super.remove();
- evictFunc.apply(analysisInfo);
- return analysisInfo;
- }
-
- public SimpleQueue(long limit, Function<T, Void> offerFunc, Function<T, Void> evictFunc, Collection<T> collection) {
- this(limit, offerFunc, evictFunc);
- if (collection != null) {
- for (T e : collection) {
- offer(e);
- }
- }
- }
-}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
index 69f6dc0dd5d..2fc6d24e305 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
@@ -30,7 +30,6 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.statistics.AnalysisInfo.AnalysisType;
import org.apache.doris.statistics.AnalysisInfo.JobType;
import org.apache.doris.statistics.AnalysisInfo.ScheduleType;
-import org.apache.doris.statistics.util.SimpleQueue;
import org.apache.doris.statistics.util.StatisticsUtil;
import com.google.common.annotations.VisibleForTesting;
@@ -331,28 +330,6 @@ public class AnalysisManagerTest {
Assertions.assertTrue(analysisManager.analysisTaskInfoMap.containsKey(3L));
}
- @Test
- public void testRecordLimit3() {
- Config.analyze_record_limit = 2;
- AnalysisManager analysisManager = new AnalysisManager();
- analysisManager.autoJobs.offer(new AnalysisInfoBuilder().setJobId(1).build());
- analysisManager.autoJobs.offer(new AnalysisInfoBuilder().setJobId(2).build());
- analysisManager.autoJobs.offer(new AnalysisInfoBuilder().setJobId(3).build());
- Assertions.assertEquals(2, analysisManager.autoJobs.size());
- }
-
- @Test
- public void testCreateSimpleQueue() {
- AnalysisManager analysisManager = new AnalysisManager();
- ArrayList<AnalysisInfo> jobs = Lists.newArrayList();
- jobs.add(new AnalysisInfoBuilder().setJobId(1).build());
- jobs.add(new AnalysisInfoBuilder().setJobId(2).build());
- SimpleQueue<AnalysisInfo> simpleQueue = analysisManager.createSimpleQueue(jobs, analysisManager);
- Assertions.assertEquals(2, simpleQueue.size());
- simpleQueue = analysisManager.createSimpleQueue(null, analysisManager);
- Assertions.assertEquals(0, simpleQueue.size());
- }
-
@Test
public void testShowAutoJobs(@Injectable ShowAnalyzeStmt stmt) {
new MockUp<ShowAnalyzeStmt>() {
@@ -404,5 +381,4 @@ public class AnalysisManagerTest {
Assertions.assertEquals(AnalysisState.FINISHED, analysisInfos.get(1).getState());
Assertions.assertEquals(AnalysisState.FAILED, analysisInfos.get(2).getState());
}
-
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 05/22: [fix](Nereids): fix enable mv rewrite is useless in dphyp (#30042)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit d4195b3cc011b524613de93a772fe0e9dd52af55
Author: 谢健 <ji...@gmail.com>
AuthorDate: Wed Jan 17 15:11:20 2024 +0800
[fix](Nereids): fix enable mv rewrite is useless in dphyp (#30042)
---
.../jobs/cascades/OptimizeGroupExpressionJob.java | 23 ++++++++++++++++------
.../org/apache/doris/nereids/rules/RuleSet.java | 1 -
2 files changed, 17 insertions(+), 7 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java
index 16cf90b786e..8c8e7e43998 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java
@@ -24,7 +24,8 @@ import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.qe.ConnectContext;
-import java.util.ArrayList;
+import com.google.common.collect.ImmutableList;
+
import java.util.Collections;
import java.util.List;
@@ -48,11 +49,6 @@ public class OptimizeGroupExpressionJob extends Job {
countJobExecutionTimesOfGroupExpressions(groupExpression);
List<Rule> implementationRules = getRuleSet().getImplementationRules();
List<Rule> explorationRules = getExplorationRules();
- ConnectContext connectContext = context.getCascadesContext().getConnectContext();
- if (connectContext.getSessionVariable().isEnableMaterializedViewRewrite()) {
- explorationRules = new ArrayList<>(explorationRules);
- explorationRules.addAll(getRuleSet().getMaterializedViewRules());
- }
for (Rule rule : explorationRules) {
if (rule.isInvalid(disableRules, groupExpression)) {
@@ -70,6 +66,13 @@ public class OptimizeGroupExpressionJob extends Job {
}
private List<Rule> getExplorationRules() {
+ return ImmutableList.<Rule>builder()
+ .addAll(getJoinRules())
+ .addAll(getMvRules())
+ .build();
+ }
+
+ private List<Rule> getJoinRules() {
boolean isDisableJoinReorder = context.getCascadesContext().getConnectContext().getSessionVariable()
.isDisableJoinReorder()
|| context.getCascadesContext().getMemo().getGroupExpressionsSize() > context.getCascadesContext()
@@ -96,4 +99,12 @@ public class OptimizeGroupExpressionJob extends Job {
return getRuleSet().getZigZagTreeJoinReorder();
}
}
+
+ private List<Rule> getMvRules() {
+ ConnectContext connectContext = context.getCascadesContext().getConnectContext();
+ if (connectContext.getSessionVariable().isEnableMaterializedViewRewrite()) {
+ return getRuleSet().getMaterializedViewRules();
+ }
+ return ImmutableList.of();
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index bb5b2f3dcf8..de0f3d518fa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -240,7 +240,6 @@ public class RuleSet {
.build();
public static final List<Rule> DPHYP_REORDER_RULES = ImmutableList.<Rule>builder()
- .addAll(MATERIALIZED_VIEW_RULES)
.add(JoinCommute.BUSHY.build())
.build();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 21/22: [fix](ut) fixed test cases failure for showCreateFunction (#28593)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit cb5fe27b5ac296b319c0dc2998a4010e3ac53b4f
Author: Nitin-Kashyap <66...@users.noreply.github.com>
AuthorDate: Wed Jan 17 21:07:25 2024 +0530
[fix](ut) fixed test cases failure for showCreateFunction (#28593)
2 tests cases (ShowCreateFunctionTest and ShowFunctionTest) have function name clash;
because of which one of the test fails to complete setup() and eventually fail for not able to create funciton.
Changed the name of the test function in 1 of the file.
---
.../org/apache/doris/analysis/ShowCreateFunctionTest.java | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCreateFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCreateFunctionTest.java
index f89f6fdbf35..30256d394de 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCreateFunctionTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCreateFunctionTest.java
@@ -34,28 +34,28 @@ public class ShowCreateFunctionTest extends TestWithFeService {
createDatabase(dbName);
useDatabase(dbName);
createFunction(
- "CREATE ALIAS FUNCTION id_masking(bigint) WITH PARAMETER(id) AS CONCAT(LEFT(id,3),'****',RIGHT(id,4));");
+ "CREATE ALIAS FUNCTION id_masking_create(bigint) WITH PARAMETER(id) AS CONCAT(LEFT(id,3),'****',RIGHT(id,4));");
createFunction(
- "CREATE GLOBAL ALIAS FUNCTION id_masking_global(bigint) WITH PARAMETER(id) AS CONCAT(LEFT(id,3),'****',RIGHT(id,4));");
+ "CREATE GLOBAL ALIAS FUNCTION id_masking_global_create(bigint) WITH PARAMETER(id) AS CONCAT(LEFT(id,3),'****',RIGHT(id,4));");
}
@Test
public void testNormal() throws Exception {
- String sql = "SHOW CREATE FUNCTION id_masking(bigint)";
+ String sql = "SHOW CREATE FUNCTION id_masking_create(bigint)";
ShowResultSet showResultSet = showCreateFunction(sql);
String showSql = showResultSet.getResultRows().get(0).get(1);
- Assertions.assertTrue(showSql.contains("CREATE ALIAS FUNCTION id_masking(BIGINT) WITH PARAMETER(id)"));
+ Assertions.assertTrue(showSql.contains("CREATE ALIAS FUNCTION id_masking_create(BIGINT) WITH PARAMETER(id)"));
}
@Test
public void testShowCreateGlobalFunction() throws Exception {
- String sql = "SHOW CREATE GLOBAL FUNCTION id_masking_global(bigint)";
+ String sql = "SHOW CREATE GLOBAL FUNCTION id_masking_global_create(bigint)";
ShowResultSet showResultSet = showCreateFunction(sql);
String showSql = showResultSet.getResultRows().get(0).get(1);
Assertions.assertTrue(
- showSql.contains("CREATE GLOBAL ALIAS FUNCTION id_masking_global(BIGINT) WITH PARAMETER(id)"));
+ showSql.contains("CREATE GLOBAL ALIAS FUNCTION id_masking_global_create(BIGINT) WITH PARAMETER(id)"));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 08/22: [test](regression-test) fix case, compatible with 3 replicas. (#29905)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 48d7c1b1eddd70b2cdb349dd6b36306874296fb8
Author: shuke <37...@users.noreply.github.com>
AuthorDate: Wed Jan 17 17:06:09 2024 +0800
[test](regression-test) fix case, compatible with 3 replicas. (#29905)
---
.../org/apache/doris/regression/suite/Suite.groovy | 27 +++++++++++++++++++++-
.../test_index_change_with_compaction.groovy | 13 ++++++++++-
.../test_index_compaction_dup_keys.groovy | 10 +++++++-
.../test_index_compaction_unique_keys.groovy | 10 +++++++-
...dex_compaction_with_multi_index_segments.groovy | 10 +++++++-
.../stream_load/test_load_to_single_tablet.groovy | 4 +++-
.../select_tablets/select_with_tablets.groovy | 3 +++
7 files changed, 71 insertions(+), 6 deletions(-)
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index f276e29a530..41a99abe7a7 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -932,5 +932,30 @@ class Suite implements GroovyInterceptable {
}
Assert.assertEquals(true, !createdTableName.isEmpty())
}
-}
+ String[][] deduplicate_tablets(String[][] tablets) {
+ def result = [:]
+
+ tablets.each { row ->
+ def tablet_id = row[0]
+ if (!result.containsKey(tablet_id)) {
+ result[tablet_id] = row
+ }
+ }
+
+ return result.values().toList()
+ }
+
+ ArrayList deduplicate_tablets(ArrayList tablets) {
+ def result = [:]
+
+ tablets.each { row ->
+ def tablet_id = row[0]
+ if (!result.containsKey(tablet_id)) {
+ result[tablet_id] = row
+ }
+ }
+
+ return result.values().toList()
+ }
+}
diff --git a/regression-test/suites/inverted_index_p0/index_change/test_index_change_with_compaction.groovy b/regression-test/suites/inverted_index_p0/index_change/test_index_change_with_compaction.groovy
index b4a42aba237..924ab0a9f4e 100644
--- a/regression-test/suites/inverted_index_p0/index_change/test_index_change_with_compaction.groovy
+++ b/regression-test/suites/inverted_index_p0/index_change/test_index_change_with_compaction.groovy
@@ -226,7 +226,18 @@ suite("test_index_change_with_compaction") {
rowCount += Integer.parseInt(rowset.split(" ")[1])
}
}
- assert (rowCount <= 8)
+
+ String[][] dedup_tablets = deduplicate_tablets(tablets)
+
+ // In the p0 testing environment, there are no expected operations such as scaling down BE (backend) services
+ // if tablets or dedup_tablets is empty, exception is thrown, and case fail
+ int replicaNum = Math.floor(tablets.size() / dedup_tablets.size())
+ if (replicaNum != 1 && replicaNum != 3)
+ {
+ assert(false);
+ }
+
+ assert (rowCount <= 8*replicaNum)
qt_select_default2 """ SELECT * FROM ${tableName} t ORDER BY user_id,date,city,age,sex,last_visit_date,last_update_date,last_visit_date_not_null,cost,max_dwell_time,min_dwell_time; """
} finally {
// try_sql("DROP TABLE IF EXISTS ${tableName}")
diff --git a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.groovy b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.groovy
index cd98f58ce4f..8bbedb97794 100644
--- a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.groovy
+++ b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.groovy
@@ -164,8 +164,16 @@ suite("test_index_compaction_dup_keys", "p0") {
//TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus
String[][] tablets = sql """ show tablets from ${tableName}; """
+ String[][] dedup_tablets = deduplicate_tablets(tablets)
+
+ // In the p0 testing environment, there are no expected operations such as scaling down BE (backend) services
+ // if tablets or dedup_tablets is empty, exception is thrown, and case fail
+ int replicaNum = Math.floor(tablets.size() / dedup_tablets.size())
+ if (replicaNum != 1 && replicaNum != 3)
+ {
+ assert(false);
+ }
- int replicaNum = 1
// before full compaction, there are 7 rowsets.
int rowsetCount = get_rowset_count.call(tablets);
assert (rowsetCount == 7 * replicaNum)
diff --git a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.groovy b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.groovy
index f4f2afb78d6..64d2ee21647 100644
--- a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.groovy
+++ b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.groovy
@@ -168,8 +168,16 @@ suite("test_index_compaction_unique_keys", "p0") {
//TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus
String[][] tablets = sql """ show tablets from ${tableName}; """
+ String[][] dedup_tablets = deduplicate_tablets(tablets)
+
+ // In the p0 testing environment, there are no expected operations such as scaling down BE (backend) services
+ // if tablets or dedup_tablets is empty, exception is thrown, and case fail
+ int replicaNum = Math.floor(tablets.size() / dedup_tablets.size())
+ if (replicaNum != 1 && replicaNum != 3)
+ {
+ assert(false);
+ }
- int replicaNum = 1
// before full compaction, there are 7 rowsets.
int rowsetCount = get_rowset_count.call(tablets);
assert (rowsetCount == 7 * replicaNum)
diff --git a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy
index 25ed93aec51..dd6e91e53b7 100644
--- a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy
+++ b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy
@@ -194,8 +194,16 @@ suite("test_index_compaction_with_multi_index_segments", "p0") {
//TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus
String[][] tablets = sql """ show tablets from ${tableName}; """
+ String[][] dedup_tablets = deduplicate_tablets(tablets)
+
+ // In the p0 testing environment, there are no expected operations such as scaling down BE (backend) services
+ // if tablets or dedup_tablets is empty, exception is thrown, and case fail
+ int replicaNum = Math.floor(tablets.size() / dedup_tablets.size())
+ if (replicaNum != 1 && replicaNum != 3)
+ {
+ assert(false);
+ }
- int replicaNum = 1
// before full compaction, there are 3 rowsets.
int rowsetCount = get_rowset_count.call(tablets)
assert (rowsetCount == 3 * replicaNum)
diff --git a/regression-test/suites/load_p0/stream_load/test_load_to_single_tablet.groovy b/regression-test/suites/load_p0/stream_load/test_load_to_single_tablet.groovy
index c35500267b9..80b25bd34c3 100644
--- a/regression-test/suites/load_p0/stream_load/test_load_to_single_tablet.groovy
+++ b/regression-test/suites/load_p0/stream_load/test_load_to_single_tablet.groovy
@@ -55,7 +55,8 @@ suite("test_load_to_single_tablet", "p0") {
sql "sync"
def totalCount = sql "select count() from ${tableName}"
assertEquals(10, totalCount[0][0])
- def res = sql "show tablets from ${tableName}"
+ String[][] res = sql "show tablets from ${tableName}"
+ res = deduplicate_tablets(res)
def tablet1 = res[0][0]
def tablet2 = res[1][0]
def tablet3 = res[2][0]
@@ -150,6 +151,7 @@ suite("test_load_to_single_tablet", "p0") {
totalCount = sql "select count() from ${tableName}"
assertEquals(10, totalCount[0][0])
res = sql "show tablets from ${tableName} partitions(p20231011, p20231012)"
+ res = deduplicate_tablets(res)
tablet1 = res[0][0]
tablet2 = res[1][0]
tablet3 = res[2][0]
diff --git a/regression-test/suites/nereids_p0/select_tablets/select_with_tablets.groovy b/regression-test/suites/nereids_p0/select_tablets/select_with_tablets.groovy
index f2c2546b024..86bc846a110 100644
--- a/regression-test/suites/nereids_p0/select_tablets/select_with_tablets.groovy
+++ b/regression-test/suites/nereids_p0/select_tablets/select_with_tablets.groovy
@@ -45,6 +45,7 @@ suite("select_with_tablets") {
order_qt_select1 """ SELECT * FROM ${table_name1} """
def res = sql """ show tablets from ${table_name1} where version = 2 """
+ res = deduplicate_tablets(res)
assertTrue(res.size() == 1)
assertTrue(res[0].size() == 21)
assertEquals("2", res[0][4])
@@ -65,6 +66,7 @@ suite("select_with_tablets") {
order_qt_select11 """ SELECT * FROM ${table_name1} PARTITION between_20_70 where id < 2"""
res = sql """ show tablets from ${table_name1} where version = 1 """
+ res = deduplicate_tablets(res)
assertTrue(res.size() == 2)
assertEquals("1", res[0][4])
assertEquals("1", res[1][4])
@@ -93,6 +95,7 @@ suite("select_with_tablets") {
order_qt_no_partition_1 """ SELECT * FROM ${table_no_partition} """
res = sql """ show tablets from ${table_no_partition} where version = 2 """
+ res = deduplicate_tablets(res)
order_qt_no_partition_2 """ SELECT * FROM ${table_no_partition} TABLET(${res[0][0]}) """
order_qt_no_partition_3 """ SELECT * FROM ${table_no_partition} TABLET(${res[1][0]}) """
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 19/22: [Performance](Join) Support all match one logic (#30019)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 77230172537b2ca28fed017487672d2d1ba2b2ed
Author: HappenLee <ha...@hotmail.com>
AuthorDate: Wed Jan 17 20:53:42 2024 +0800
[Performance](Join) Support all match one logic (#30019)
Support all match one logic
---
be/src/vec/exec/join/process_hash_table_probe.h | 4 ++--
.../vec/exec/join/process_hash_table_probe_impl.h | 24 ++++++++++++++--------
2 files changed, 18 insertions(+), 10 deletions(-)
diff --git a/be/src/vec/exec/join/process_hash_table_probe.h b/be/src/vec/exec/join/process_hash_table_probe.h
index 295317517d6..4b3140b7b59 100644
--- a/be/src/vec/exec/join/process_hash_table_probe.h
+++ b/be/src/vec/exec/join/process_hash_table_probe.h
@@ -47,8 +47,8 @@ struct ProcessHashTableProbe {
int size, bool have_other_join_conjunct);
void probe_side_output_column(MutableColumns& mcol, const std::vector<bool>& output_slot_flags,
- int size, int last_probe_index, size_t probe_size,
- bool all_match_one, bool have_other_join_conjunct);
+ int size, int last_probe_index, bool all_match_one,
+ bool have_other_join_conjunct);
template <bool need_null_map_for_probe, bool ignore_null, typename HashTableType>
Status process(HashTableType& hash_table_ctx, ConstNullMapPtr null_map,
diff --git a/be/src/vec/exec/join/process_hash_table_probe_impl.h b/be/src/vec/exec/join/process_hash_table_probe_impl.h
index a8ef9c06484..4f950f876cf 100644
--- a/be/src/vec/exec/join/process_hash_table_probe_impl.h
+++ b/be/src/vec/exec/join/process_hash_table_probe_impl.h
@@ -97,15 +97,14 @@ void ProcessHashTableProbe<JoinOpType, Parent>::build_side_output_column(
template <int JoinOpType, typename Parent>
void ProcessHashTableProbe<JoinOpType, Parent>::probe_side_output_column(
MutableColumns& mcol, const std::vector<bool>& output_slot_flags, int size,
- int last_probe_index, size_t probe_size, bool all_match_one,
- bool have_other_join_conjunct) {
+ int last_probe_index, bool all_match_one, bool have_other_join_conjunct) {
SCOPED_TIMER(_probe_side_output_timer);
auto& probe_block = _parent->_probe_block;
for (int i = 0; i < output_slot_flags.size(); ++i) {
if (output_slot_flags[i]) {
auto& column = probe_block.get_by_position(i).column;
if (all_match_one) {
- mcol[i]->insert_range_from(*column, last_probe_index, probe_size);
+ mcol[i]->insert_range_from(*column, last_probe_index, size);
} else {
column->replicate(_probe_indexs.data(), size, *mcol[i]);
}
@@ -168,8 +167,6 @@ Status ProcessHashTableProbe<JoinOpType, Parent>::do_process(HashTableType& hash
auto& mcol = mutable_block.mutable_columns();
int current_offset = 0;
- bool all_match_one = false;
- size_t probe_size = 0;
std::unique_ptr<ColumnFilterHelper> mark_column;
if (is_mark_join) {
@@ -188,16 +185,27 @@ Status ProcessHashTableProbe<JoinOpType, Parent>::do_process(HashTableType& hash
probe_index = new_probe_idx;
build_index = new_build_idx;
current_offset = new_current_offset;
- probe_size = probe_index - last_probe_index;
}
build_side_output_column(mcol, *_right_output_slot_flags, current_offset, with_other_conjuncts);
if constexpr (with_other_conjuncts || (JoinOpType != TJoinOp::RIGHT_SEMI_JOIN &&
JoinOpType != TJoinOp::RIGHT_ANTI_JOIN)) {
+ auto check_all_match_one = [](const std::vector<uint32_t>& vecs, uint32_t probe_idx,
+ int size) {
+ if (size < 1 || vecs[0] != probe_idx) return false;
+ for (int i = 1; i < size; i++) {
+ if (vecs[i] - vecs[i - 1] != 1) {
+ return false;
+ }
+ }
+ return true;
+ };
+
RETURN_IF_CATCH_EXCEPTION(probe_side_output_column(
- mcol, *_left_output_slot_flags, current_offset, last_probe_index, probe_size,
- all_match_one, with_other_conjuncts));
+ mcol, *_left_output_slot_flags, current_offset, last_probe_index,
+ check_all_match_one(_probe_indexs, last_probe_index, current_offset),
+ with_other_conjuncts));
}
output_block->swap(mutable_block.to_block());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 09/22: [config](Nereids): add SessionVariable to control nereids timeout (#30048)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit dcc9cf9dd3db8e882655cce12a3a215a2176f1e5
Author: jakevin <ja...@gmail.com>
AuthorDate: Wed Jan 17 17:12:04 2024 +0800
[config](Nereids): add SessionVariable to control nereids timeout (#30048)
---
.../doris/nereids/jobs/scheduler/SimpleJobScheduler.java | 12 ++++++++----
.../src/main/java/org/apache/doris/qe/SessionVariable.java | 3 +++
2 files changed, 11 insertions(+), 4 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/SimpleJobScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/SimpleJobScheduler.java
index c52e2e90f13..ec751bdab2d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/SimpleJobScheduler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/SimpleJobScheduler.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.jobs.scheduler;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.jobs.Job;
+import org.apache.doris.qe.SessionVariable;
import java.util.concurrent.TimeUnit;
@@ -29,11 +30,14 @@ public class SimpleJobScheduler implements JobScheduler {
@Override
public void executeJobPool(ScheduleContext scheduleContext) {
JobPool pool = scheduleContext.getJobPool();
+ CascadesContext context = (CascadesContext) scheduleContext;
+ SessionVariable sessionVariable = context.getConnectContext().getSessionVariable();
while (!pool.isEmpty()) {
- CascadesContext context = (CascadesContext) scheduleContext;
- if (context.getConnectContext().getSessionVariable().enableNereidsTimeout
- && context.getStatementContext().getStopwatch().elapsed(TimeUnit.MILLISECONDS) > 5000) {
- throw new RuntimeException("Nereids cost too much time ( > 5s )");
+ if (sessionVariable.enableNereidsTimeout
+ && context.getStatementContext().getStopwatch().elapsed(TimeUnit.MILLISECONDS)
+ > sessionVariable.nereidsTimeoutSecond * 1000L) {
+ throw new RuntimeException(
+ "Nereids cost too much time ( > " + sessionVariable.nereidsTimeoutSecond + "s )");
}
Job job = pool.pop();
job.execute();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index e1dd554c0dd..5a78e49009b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -1155,6 +1155,9 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = ENABLE_NEREIDS_TIMEOUT, needForward = true)
public boolean enableNereidsTimeout = true;
+ @VariableMgr.VarAttr(name = "nereids_timeout_second", needForward = true)
+ public int nereidsTimeoutSecond = 5;
+
@VariableMgr.VarAttr(name = ENABLE_PUSH_DOWN_NO_GROUP_AGG)
public boolean enablePushDownNoGroupAgg = true;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 18/22: [Fix](typo) Fix group commit regression test typo (#30057)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 5fcdf63a3947d07da6eea647fa6503d234189a42
Author: abmdocrt <Yu...@gmail.com>
AuthorDate: Wed Jan 17 20:37:29 2024 +0800
[Fix](typo) Fix group commit regression test typo (#30057)
---
.../fault_injection_p0/test_low_wal_disk_space_fault_injection.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/regression-test/suites/fault_injection_p0/test_low_wal_disk_space_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_low_wal_disk_space_fault_injection.groovy
index 815bafe3d4b..0b7cab481f7 100644
--- a/regression-test/suites/fault_injection_p0/test_low_wal_disk_space_fault_injection.groovy
+++ b/regression-test/suites/fault_injection_p0/test_low_wal_disk_space_fault_injection.groovy
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-suite("test_wal_mem_back_pressure_time_out_fault_injection","nonConcurrent") {
+suite("test_low_wal_disk_space_fault_injection","nonConcurrent") {
def tableName = "wal_test"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 16/22: [minor](Nereids): remove useless method isMysqlCompatibleDatabase (#30064)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 6c9ec7b479c6c77da4298c117be66171a98d49c2
Author: jakevin <ja...@gmail.com>
AuthorDate: Wed Jan 17 19:20:19 2024 +0800
[minor](Nereids): remove useless method isMysqlCompatibleDatabase (#30064)
---
.../src/main/java/org/apache/doris/analysis/DropDbStmt.java | 4 ++--
.../src/main/java/org/apache/doris/catalog/Database.java | 13 +++----------
.../src/main/java/org/apache/doris/catalog/DatabaseIf.java | 2 +-
.../org/apache/doris/catalog/MysqlCompatibleDatabase.java | 5 -----
.../src/main/java/org/apache/doris/clone/TabletChecker.java | 3 ++-
.../main/java/org/apache/doris/datasource/CatalogIf.java | 6 +++---
.../java/org/apache/doris/datasource/InternalCatalog.java | 10 +++++-----
.../main/java/org/apache/doris/httpv2/rest/ShowAction.java | 5 +++--
.../apache/doris/master/PartitionInMemoryInfoCollector.java | 3 ++-
.../doris/transaction/DbUsedDataQuotaInfoCollector.java | 3 ++-
.../java/org/apache/doris/clone/RebalancerTestUtil.java | 3 ++-
11 files changed, 25 insertions(+), 32 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropDbStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropDbStmt.java
index 9b4f58efbf0..08a1e630eb5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropDbStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropDbStmt.java
@@ -17,9 +17,9 @@
package org.apache.doris.analysis;
-import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.MysqlCompatibleDatabase;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
@@ -62,7 +62,7 @@ public class DropDbStmt extends DdlStmt {
InternalDatabaseUtil.checkDatabase(dbName, ConnectContext.get());
// Don't allow to drop mysql compatible databases
DatabaseIf db = Env.getCurrentInternalCatalog().getDbNullable(dbName);
- if (db != null && (db instanceof Database) && ((Database) db).isMysqlCompatibleDatabase()) {
+ if (db != null && db instanceof MysqlCompatibleDatabase) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_DBACCESS_DENIED_ERROR,
analyzer.getQualifiedUser(), dbName);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
index 71e61d5791c..96db31af430 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
@@ -84,14 +84,14 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>
private long id;
@SerializedName(value = "fullQualifiedName")
private volatile String fullQualifiedName;
- private ReentrantReadWriteLock rwLock;
+ private final ReentrantReadWriteLock rwLock;
// table family group map
- private Map<Long, Table> idToTable;
+ private final Map<Long, Table> idToTable;
@SerializedName(value = "nameToTable")
private Map<String, Table> nameToTable;
// table name lower cast -> table name
- private Map<String, String> lowerCaseToTableName;
+ private final Map<String, String> lowerCaseToTableName;
// user define function
@SerializedName(value = "name2Function")
@@ -893,11 +893,4 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>
public String toString() {
return toJson();
}
-
- // Return ture if database is created for mysql compatibility.
- // Currently, we have two dbs that are created for this purpose, InformationSchemaDb and MysqlDb,
- public boolean isMysqlCompatibleDatabase() {
- return false;
- }
-
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java
index 3ba29573374..e80f335be17 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java
@@ -273,5 +273,5 @@ public interface DatabaseIf<T extends TableIf> {
return -1L;
}
- public Map<Long, TableIf> getIdToTable();
+ Map<Long, TableIf> getIdToTable();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlCompatibleDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlCompatibleDatabase.java
index 1d4aa62dc9d..12d915321ed 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlCompatibleDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlCompatibleDatabase.java
@@ -77,11 +77,6 @@ public abstract class MysqlCompatibleDatabase extends Database {
throw new IOException("Not support.");
}
- @Override
- public boolean isMysqlCompatibleDatabase() {
- return true;
- }
-
/**
* This method must be re-implemented since {@link Env#createView(CreateViewStmt)}
* will call this method. And create view should not succeed under this database.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java
index c6f6be3d460..66497e7d18e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java
@@ -23,6 +23,7 @@ import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
+import org.apache.doris.catalog.MysqlCompatibleDatabase;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.Partition.PartitionState;
@@ -280,7 +281,7 @@ public class TabletChecker extends MasterDaemon {
continue;
}
- if (db.isMysqlCompatibleDatabase()) {
+ if (db instanceof MysqlCompatibleDatabase) {
continue;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
index c8f3033c668..7fd240aa8d0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
@@ -172,9 +172,9 @@ public interface CatalogIf<T extends DatabaseIf> {
}
// Return a copy of all db collection.
- public Collection<DatabaseIf<? extends TableIf>> getAllDbs();
+ Collection<DatabaseIf<? extends TableIf>> getAllDbs();
- public boolean enableAutoAnalyze();
+ boolean enableAutoAnalyze();
- public ConcurrentHashMap<Long, DatabaseIf> getIdToDb();
+ ConcurrentHashMap<Long, DatabaseIf> getIdToDb();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 3212d18c1d3..47ea998bcb3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -839,8 +839,8 @@ public class InternalCatalog implements CatalogIf<Database> {
LOG.info("begin to drop table: {} from db: {}, is force: {}", tableName, dbName, stmt.isForceDrop());
// check database
- Database db = (Database) getDbOrDdlException(dbName);
- if (db.isMysqlCompatibleDatabase()) {
+ Database db = getDbOrDdlException(dbName);
+ if (db instanceof MysqlCompatibleDatabase) {
throw new DdlException("Drop table from this database is not allowed.");
}
@@ -1066,7 +1066,7 @@ public class InternalCatalog implements CatalogIf<Database> {
// check if db exists
Database db = getDbOrDdlException(dbName);
// InfoSchemaDb and MysqlDb can not create table manually
- if (db.isMysqlCompatibleDatabase()) {
+ if (db instanceof MysqlCompatibleDatabase) {
ErrorReport.reportDdlException(ErrorCode.ERR_CANT_CREATE_TABLE, tableName,
ErrorCode.ERR_CANT_CREATE_TABLE.getCode(), "not supported create table in this database");
}
@@ -3227,7 +3227,7 @@ public class InternalCatalog implements CatalogIf<Database> {
for (Map.Entry<Long, Database> entry : idToDb.entrySet()) {
Database db = entry.getValue();
// Don't write internal database meta.
- if (!db.isMysqlCompatibleDatabase()) {
+ if (!(db instanceof MysqlCompatibleDatabase)) {
checksum ^= entry.getKey();
db.write(dos);
}
@@ -3246,7 +3246,7 @@ public class InternalCatalog implements CatalogIf<Database> {
Database dbPrev = fullNameToDb.get(db.getFullName());
if (dbPrev != null) {
String errMsg;
- if (dbPrev.isMysqlCompatibleDatabase() || db.isMysqlCompatibleDatabase()) {
+ if (dbPrev instanceof MysqlCompatibleDatabase || db instanceof MysqlCompatibleDatabase) {
errMsg = String.format(
"Mysql compatibility problem, previous checkpoint already has a database with full name "
+ "%s. If its name is mysql, try to add mysqldb_replace_name=\"mysql_comp\" in fe.conf.",
diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java
index 6db2a7e7b51..0aa5dd54d78 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java
@@ -20,6 +20,7 @@ package org.apache.doris.httpv2.rest;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.MysqlCompatibleDatabase;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf.TableType;
@@ -199,7 +200,7 @@ public class ShowAction extends RestBaseController {
} else {
for (long dbId : Env.getCurrentInternalCatalog().getDbIds()) {
DatabaseIf db = Env.getCurrentInternalCatalog().getDbNullable(dbId);
- if (db == null || !(db instanceof Database) || ((Database) db).isMysqlCompatibleDatabase()) {
+ if (db == null || !(db instanceof Database) || db instanceof MysqlCompatibleDatabase) {
continue;
}
totalSize += getDataSizeOfDatabase(db);
@@ -232,7 +233,7 @@ public class ShowAction extends RestBaseController {
} else {
for (long dbId : Env.getCurrentInternalCatalog().getDbIds()) {
DatabaseIf db = Env.getCurrentInternalCatalog().getDbNullable(dbId);
- if (db == null || !(db instanceof Database) || ((Database) db).isMysqlCompatibleDatabase()) {
+ if (db == null || !(db instanceof Database) || ((Database) db) instanceof MysqlCompatibleDatabase) {
continue;
}
Map<String, Long> tablesEntry = getDataSizeOfTables(db, tableName, singleReplicaBool);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/PartitionInMemoryInfoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/master/PartitionInMemoryInfoCollector.java
index 22f2d41cfad..77ed5829799 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/PartitionInMemoryInfoCollector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/PartitionInMemoryInfoCollector.java
@@ -19,6 +19,7 @@ package org.apache.doris.master;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.MysqlCompatibleDatabase;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.Table;
@@ -56,7 +57,7 @@ public class PartitionInMemoryInfoCollector extends MasterDaemon {
LOG.warn("Database [" + dbId + "] does not exist, skip to update database used data quota");
continue;
}
- if (db.isMysqlCompatibleDatabase()) {
+ if (db instanceof MysqlCompatibleDatabase) {
continue;
}
try {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/DbUsedDataQuotaInfoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/DbUsedDataQuotaInfoCollector.java
index 87d509bbb4d..3a713bdc63f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/transaction/DbUsedDataQuotaInfoCollector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/DbUsedDataQuotaInfoCollector.java
@@ -19,6 +19,7 @@ package org.apache.doris.transaction;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.MysqlCompatibleDatabase;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.util.MasterDaemon;
@@ -50,7 +51,7 @@ public class DbUsedDataQuotaInfoCollector extends MasterDaemon {
LOG.warn("Database [" + dbId + "] does not exist, skip to update database used data quota");
continue;
}
- if (db.isMysqlCompatibleDatabase()) {
+ if (db instanceof MysqlCompatibleDatabase) {
continue;
}
try {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/clone/RebalancerTestUtil.java b/fe/fe-core/src/test/java/org/apache/doris/clone/RebalancerTestUtil.java
index da03d42a644..063faf2d3b2 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/clone/RebalancerTestUtil.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/clone/RebalancerTestUtil.java
@@ -21,6 +21,7 @@ import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.DiskInfo;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MaterializedIndex;
+import org.apache.doris.catalog.MysqlCompatibleDatabase;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.Replica;
@@ -145,7 +146,7 @@ public class RebalancerTestUtil {
continue;
}
- if (db.isMysqlCompatibleDatabase()) {
+ if (db instanceof MysqlCompatibleDatabase) {
continue;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 12/22: [Fix](compile) fix compile failure on newest clang (#30056)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 94cbd69ccae629b1cec6c7177cfdb430261d7e4b
Author: zclllyybb <zh...@selectdb.com>
AuthorDate: Wed Jan 17 17:39:31 2024 +0800
[Fix](compile) fix compile failure on newest clang (#30056)
fix compile failure on newest clang
---
be/src/service/backend_service.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/be/src/service/backend_service.cpp b/be/src/service/backend_service.cpp
index fb98d60dce4..72a89f884b2 100644
--- a/be/src/service/backend_service.cpp
+++ b/be/src/service/backend_service.cpp
@@ -787,14 +787,14 @@ void BackendService::ingest_binlog(TIngestBinlogResult& result,
bool is_async = (_ingest_binlog_workers != nullptr);
result.__set_is_async(is_async);
- auto ingest_binlog_func = [=, tstatus = &tstatus]() {
+ auto ingest_binlog_func = [=, this, tstatus = &tstatus]() {
IngestBinlogArg ingest_binlog_arg = {
.txn_id = txn_id,
.partition_id = partition_id,
.local_tablet_id = local_tablet_id,
.local_tablet = local_tablet,
- .request = std::move(request),
+ .request = request,
.tstatus = is_async ? nullptr : tstatus,
};
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 17/22: [cleanup](insert-into) clean up some insert into log (#30063)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit d0977824e934664bf2406cd27c6c31012416d463
Author: HHoflittlefish777 <77...@users.noreply.github.com>
AuthorDate: Wed Jan 17 19:59:14 2024 +0800
[cleanup](insert-into) clean up some insert into log (#30063)
---
be/src/pipeline/exec/union_source_operator.cpp | 6 ------
be/src/runtime/fragment_mgr.cpp | 5 -----
be/src/vec/exec/vunion_node.cpp | 6 ------
.../src/main/java/org/apache/doris/qe/Coordinator.java | 14 +++-----------
regression-test/pipeline/p0/conf/be.conf | 2 +-
5 files changed, 4 insertions(+), 29 deletions(-)
diff --git a/be/src/pipeline/exec/union_source_operator.cpp b/be/src/pipeline/exec/union_source_operator.cpp
index 31806d7242c..e8ef1ba7207 100644
--- a/be/src/pipeline/exec/union_source_operator.cpp
+++ b/be/src/pipeline/exec/union_source_operator.cpp
@@ -223,9 +223,6 @@ Status UnionSourceOperatorX::get_next_const(RuntimeState* state, vectorized::Blo
&result_list[i]));
}
tmp_block.erase_not_in(result_list);
- VLOG_ROW << "query id: " << print_id(state->query_id())
- << ", instance id: " << print_id(state->fragment_instance_id())
- << ", tmp_block rows: " << tmp_block.rows();
if (tmp_block.rows() > 0) {
RETURN_IF_ERROR(mblock.merge(tmp_block));
tmp_block.clear();
@@ -235,9 +232,6 @@ Status UnionSourceOperatorX::get_next_const(RuntimeState* state, vectorized::Blo
// some insert query like "insert into string_test select 1, repeat('a', 1024 * 1024);"
// the const expr will be in output expr cause the union node return a empty block. so here we
// need add one row to make sure the union node exec const expr return at least one row
- VLOG_ROW << "query id: " << print_id(state->query_id())
- << ", instance id: " << print_id(state->fragment_instance_id())
- << ", tmp_block rows: " << block->rows();
if (block->rows() == 0) {
block->insert({vectorized::ColumnUInt8::create(1),
std::make_shared<vectorized::DataTypeUInt8>(), ""});
diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp
index 7d3dba81020..0fbaacb4fa2 100644
--- a/be/src/runtime/fragment_mgr.cpp
+++ b/be/src/runtime/fragment_mgr.cpp
@@ -322,11 +322,6 @@ void FragmentMgr::coordinator_callback(const ReportStatusRequest& req) {
params.load_counters.emplace(s_dpp_normal_all, std::to_string(num_rows_load_success));
params.load_counters.emplace(s_dpp_abnormal_all, std::to_string(num_rows_load_filtered));
params.load_counters.emplace(s_unselected_rows, std::to_string(num_rows_load_unselected));
- LOG(INFO) << "execute coordinator callback, query id: " << print_id(req.query_id)
- << ", instance id: " << print_id(req.fragment_instance_id)
- << ", num_rows_load_success: " << num_rows_load_success
- << ", num_rows_load_filtered: " << num_rows_load_filtered
- << ", num_rows_load_unselected: " << num_rows_load_unselected;
if (!req.runtime_state->get_error_log_file_path().empty()) {
params.__set_tracking_url(
diff --git a/be/src/vec/exec/vunion_node.cpp b/be/src/vec/exec/vunion_node.cpp
index 77f9c0a40f2..8ee258ca056 100644
--- a/be/src/vec/exec/vunion_node.cpp
+++ b/be/src/vec/exec/vunion_node.cpp
@@ -229,18 +229,12 @@ Status VUnionNode::get_next_const(RuntimeState* state, Block* block) {
&result_list[i]));
}
tmp_block.erase_not_in(result_list);
- VLOG_ROW << "query id: " << print_id(state->query_id())
- << ", instance id: " << print_id(state->fragment_instance_id())
- << ", tmp_block rows: " << tmp_block.rows();
if (tmp_block.rows() > 0) {
RETURN_IF_ERROR(mblock.merge(tmp_block));
tmp_block.clear();
}
}
block->set_columns(std::move(mblock.mutable_columns()));
- VLOG_ROW << "query id: " << print_id(state->query_id())
- << ", instance id: " << print_id(state->fragment_instance_id())
- << ", block rows: " << block->rows();
// some insert query like "insert into string_test select 1, repeat('a', 1024 * 1024);"
// the const expr will be in output expr cause the union node return a empty block. so here we
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 165583b9e68..2f58e560740 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -688,12 +688,6 @@ public class Coordinator implements CoordInterface {
} else {
executionProfile.markInstances(instanceIds);
}
- StringBuilder ids = new StringBuilder();
- for (TUniqueId instanceId : instanceIds) {
- ids.append(DebugUtil.printId(instanceId));
- ids.append(", ");
- }
- LOG.info("mark instances: {}", ids.toString());
if (enablePipelineEngine) {
sendPipelineCtx();
@@ -2494,7 +2488,7 @@ public class Coordinator implements CoordInterface {
Preconditions.checkArgument(params.isSetDetailedReport());
if (ctx.done) {
- LOG.info("Query {} fragment {} is marked done",
+ LOG.debug("Query {} fragment {} is marked done",
DebugUtil.printId(queryId), ctx.profileFragmentId);
executionProfile.markOneFragmentDone(ctx.profileFragmentId);
}
@@ -2552,11 +2546,11 @@ public class Coordinator implements CoordInterface {
if (params.isSetErrorTabletInfos()) {
updateErrorTabletInfos(params.getErrorTabletInfos());
}
- LOG.info("Query {} instance {} is marked done",
+ LOG.debug("Query {} instance {} is marked done",
DebugUtil.printId(queryId), DebugUtil.printId(params.getFragmentInstanceId()));
executionProfile.markOneInstanceDone(params.getFragmentInstanceId());
} else {
- LOG.info("Query {} instance {} is not marked done",
+ LOG.debug("Query {} instance {} is not marked done",
DebugUtil.printId(queryId), DebugUtil.printId(params.getFragmentInstanceId()));
}
} else {
@@ -2623,8 +2617,6 @@ public class Coordinator implements CoordInterface {
if (params.isSetErrorTabletInfos()) {
updateErrorTabletInfos(params.getErrorTabletInfos());
}
- LOG.info("Query {} instance {} is marked done",
- DebugUtil.printId(queryId), DebugUtil.printId(params.getFragmentInstanceId()));
executionProfile.markOneInstanceDone(params.getFragmentInstanceId());
}
}
diff --git a/regression-test/pipeline/p0/conf/be.conf b/regression-test/pipeline/p0/conf/be.conf
index c2c09ec89dd..d562e864a84 100644
--- a/regression-test/pipeline/p0/conf/be.conf
+++ b/regression-test/pipeline/p0/conf/be.conf
@@ -59,7 +59,7 @@ chunk_reserved_bytes_limit = 134217728
# sys_log_dir = ${DORIS_HOME}/log
# sys_log_roll_mode = SIZE-MB-1024
# sys_log_roll_num = 10
-sys_log_verbose_modules = fragment_mgr,vunion_node,union_source_operator
+# sys_log_verbose_modules = *
log_buffer_level = -1
enable_stream_load_record = true
# palo_cgroups
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 02/22: [fix](multi-table-load) fix multi table load can not finish (#29957)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 6ef9ed08aadcf263c4467ea87bccddfa81696a78
Author: HHoflittlefish777 <77...@users.noreply.github.com>
AuthorDate: Wed Jan 17 13:57:53 2024 +0800
[fix](multi-table-load) fix multi table load can not finish (#29957)
---
be/src/io/fs/multi_table_pipe.cpp | 32 ++++++++++++----------
be/src/io/fs/multi_table_pipe.h | 19 ++++++++++---
.../routine_load/routine_load_task_executor.cpp | 2 +-
3 files changed, 33 insertions(+), 20 deletions(-)
diff --git a/be/src/io/fs/multi_table_pipe.cpp b/be/src/io/fs/multi_table_pipe.cpp
index 36976a23972..916f8151739 100644
--- a/be/src/io/fs/multi_table_pipe.cpp
+++ b/be/src/io/fs/multi_table_pipe.cpp
@@ -209,7 +209,7 @@ Status MultiTablePipe::exec_plans(ExecEnv* exec_env, std::vector<ExecParam> para
_unplanned_pipes.size(), _planned_pipes.size(), params.size());
_unplanned_pipes.clear();
- _inflight_plan_cnt += params.size();
+ _inflight_cnt += params.size();
for (auto& plan : params) {
if (!plan.__isset.table_name ||
_planned_pipes.find(plan.table_name) == _planned_pipes.end()) {
@@ -263,20 +263,9 @@ Status MultiTablePipe::exec_plans(ExecEnv* exec_env, std::vector<ExecParam> para
_status = *status;
}
- --_inflight_plan_cnt;
- if (_inflight_plan_cnt == 0 && is_consume_finished()) {
- _ctx->number_total_rows = _number_total_rows;
- _ctx->number_loaded_rows = _number_loaded_rows;
- _ctx->number_filtered_rows = _number_filtered_rows;
- _ctx->number_unselected_rows = _number_unselected_rows;
- _ctx->commit_infos = _tablet_commit_infos;
- LOG(INFO) << "all plan for multi-table load complete. number_total_rows="
- << _ctx->number_total_rows
- << " number_loaded_rows=" << _ctx->number_loaded_rows
- << " number_filtered_rows=" << _ctx->number_filtered_rows
- << " number_unselected_rows=" << _ctx->number_unselected_rows;
- _ctx->promise.set_value(
- _status); // when all done, finish the routine load task
+ auto inflight_cnt = _inflight_cnt.fetch_sub(1);
+ if (inflight_cnt == 1 && is_consume_finished()) {
+ _handle_consumer_finished();
}
}));
}
@@ -303,6 +292,19 @@ Status MultiTablePipe::exec_plans(ExecEnv* exec_env, std::vector<ExecParam> para
#endif
+void MultiTablePipe::_handle_consumer_finished() {
+ _ctx->number_total_rows = _number_total_rows;
+ _ctx->number_loaded_rows = _number_loaded_rows;
+ _ctx->number_filtered_rows = _number_filtered_rows;
+ _ctx->number_unselected_rows = _number_unselected_rows;
+ _ctx->commit_infos = _tablet_commit_infos;
+ LOG(INFO) << "all plan for multi-table load complete. number_total_rows="
+ << _ctx->number_total_rows << " number_loaded_rows=" << _ctx->number_loaded_rows
+ << " number_filtered_rows=" << _ctx->number_filtered_rows
+ << " number_unselected_rows=" << _ctx->number_unselected_rows;
+ _ctx->promise.set_value(_status); // when all done, finish the routine load task
+}
+
Status MultiTablePipe::put_pipe(const TUniqueId& pipe_id,
std::shared_ptr<io::StreamLoadPipe> pipe) {
std::lock_guard<std::mutex> l(_pipe_map_lock);
diff --git a/be/src/io/fs/multi_table_pipe.h b/be/src/io/fs/multi_table_pipe.h
index 694794638af..36f6ec68b17 100644
--- a/be/src/io/fs/multi_table_pipe.h
+++ b/be/src/io/fs/multi_table_pipe.h
@@ -46,7 +46,13 @@ public:
// request and execute plans for unplanned pipes
Status request_and_exec_plans();
- void set_consume_finished() { _consume_finished.store(true, std::memory_order_release); }
+ void handle_consume_finished() {
+ _set_consume_finished();
+ auto inflight_cnt = _inflight_cnt.fetch_sub(1);
+ if (inflight_cnt == 1) {
+ _handle_consumer_finished();
+ }
+ }
bool is_consume_finished() { return _consume_finished.load(std::memory_order_acquire); }
@@ -71,25 +77,30 @@ private:
template <typename ExecParam>
Status exec_plans(ExecEnv* exec_env, std::vector<ExecParam> params);
+ void _set_consume_finished() { _consume_finished.store(true, std::memory_order_release); }
+
+ void _handle_consumer_finished();
+
private:
std::unordered_map<std::string /*table*/, KafkaConsumerPipePtr> _planned_pipes;
std::unordered_map<std::string /*table*/, KafkaConsumerPipePtr> _unplanned_pipes;
std::atomic<uint64_t> _unplanned_row_cnt {0}; // trigger plan request when exceed threshold
- std::atomic<uint64_t> _inflight_plan_cnt {0}; // how many plan fragment are executing?
+ // inflight count, when it is zero, means consume and all plans is finished
+ std::atomic<uint64_t> _inflight_cnt {1};
std::atomic<bool> _consume_finished {false};
// note: Use raw pointer here to avoid cycle reference with StreamLoadContext.
// Life cycle of MultiTablePipe is under control of StreamLoadContext, which means StreamLoadContext is created
// before NultiTablePipe and released after it. It is safe to use raw pointer here.
StreamLoadContext* _ctx = nullptr;
Status _status; // save the first error status of all executing plan fragment
-#ifndef BE_TEST
+
std::mutex _tablet_commit_infos_lock;
std::vector<TTabletCommitInfo> _tablet_commit_infos; // collect from each plan fragment
std::atomic<int64_t> _number_total_rows {0};
std::atomic<int64_t> _number_loaded_rows {0};
std::atomic<int64_t> _number_filtered_rows {0};
std::atomic<int64_t> _number_unselected_rows {0};
-#endif
+
std::mutex _pipe_map_lock;
std::unordered_map<TUniqueId /*instance id*/, std::shared_ptr<io::StreamLoadPipe>> _pipe_map;
diff --git a/be/src/runtime/routine_load/routine_load_task_executor.cpp b/be/src/runtime/routine_load/routine_load_task_executor.cpp
index dc6b855cd5a..d22f2bb4a8c 100644
--- a/be/src/runtime/routine_load/routine_load_task_executor.cpp
+++ b/be/src/runtime/routine_load/routine_load_task_executor.cpp
@@ -350,7 +350,7 @@ void RoutineLoadTaskExecutor::exec_task(std::shared_ptr<StreamLoadContext> ctx,
HANDLE_ERROR(multi_table_pipe->request_and_exec_plans(),
"multi tables task executes plan error");
// need memory order
- multi_table_pipe->set_consume_finished();
+ multi_table_pipe->handle_consume_finished();
HANDLE_ERROR(kafka_pipe->finish(), "finish multi table task failed");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org