You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by li...@apache.org on 2024/03/18 08:42:40 UTC
(doris) branch high-priority-column updated (3fb4b214172 -> 6ad12223884)
This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a change to branch high-priority-column
in repository https://gitbox.apache.org/repos/asf/doris.git
discard 3fb4b214172 refactor2 (#32278)
discard e76be97881b Refactor. (#32273)
discard c707a66a243 support window (#32094)
discard 4d426f6dbaa Check column health value earlier, show job priority. (#32064)
discard f74a289e2f2 Support show auto analyze pending jobs. (#31926)
discard 8bccfb5cc15 Support follower sync query columns to master. (#31859)
discard b8f61c4b6a7 Support column level health value. (#31794)
discard 452646e86d1 High priority queue and map. (#31509)
discard 32184b35cb4 Collect high priority columns. (#31235)
add cd5ba83e46f [fix](jdbc catalog) fix get doris column info when lower case (#32190)
add c9d6b2e9b5b [feature](merge-cloud) Add cloud copy load job meta definition (#32182)
add 7d778e035d0 [Enhancement](mor) Add unique mor table min max push down case #32196
add fe093a054b1 [fix](nereids)move SimplifyAggGroupBy rule into analyze phase (#32186)
add 601a62c2c4f [test](Nereids) add ckbench shape test (#32191)
add eb6e2fc264d [doc](auto-inc) fix example sql (#32206)
add 72a133c8ab2 [fix](iceberg) fix error when query iceberg table using date predicate (#32194)
add 3ae4243738e [chore](test) let regression test work well with nereids (#32199)
add 65945a0c262 [opt](Nereids) support cast agg state type as legacy planner (#32198)
add bd330a599fe [enhance](io) Set segment file size to rowset meta (#31853)
add 9450f5ede20 [fix](Operator) RepeatNode does not handle empty expressions. (#32112)
add d79c8dc2452 [fix](test) ckbench shape test failed because parallel merge (#32224)
add 73998920721 [opt](fe) Optimize calculate load job num metric in FE (#31952)
add 6d1080f63d3 [typo](docs) add additional explanation to the repeat function (#32158)
add 24e17a5d19d [fix](function) fixed the get_json_string function (#32150)
add 2e3d35f4f37 [refactor](rename) rename task group to workload group in be (#32204)
add ace77925aa9 [Refactor](executor)Add information_schema.workload_groups (#32195)
add 26369306009 [feature](Nereids) support agg state type in create table (#32171)
add ee0a461f966 [bugfix]json_length() BE crash fix (#32145)
add 60f97abb1f9 [fix](jdbc catalog) fix jdbc-connector coredump as get env return nullptr (#32217)
add 2815f977d3b [exec](runtimefilter) support null aware in runtime filter (#32152)
add aafd28fc015 [feature](hudi) support hudi incremental read (#32052)
add aa53eb3d42e [Fix](trino-connector) nereids supports trino-connector (#32166)
add 1a9db84c8b5 [regression](framework) add new config caseNamePrefix (#32229)
add a18845b4bc6 [fix](test) ckbench shape unstable since topn_opt_limit_threshold in fuzzy list (#32256)
add 8fdd809a7b8 [bug](s3) Fix object data is overwritten by empty object Fix wrong empty s3 file (#32252)
add cc99b496c1d [fix](ip) conversion of ipv4 or ipv6 to arrow type #32240
add b7d3173692b [opt](Nereids) opt distinct agg without group by key plan #32236
add bab06e1504c [enhancement](test) opt the unique model by modify value (#32043)
add b4377911fdb [doc](mtmv)fix doc char2 bug (#32271)
add e28d2abe86f [Enhancement](inverted index) support inverted index format v2 (#30145)
add 20c10ce8089 [opt](Nereids) remove group by key eliminate rule option (#32238)
add eeda6f19b9c [Bug](top-n) do not get runtime predicate when predicate not initialized (#32208)
add eb9079044b3 [improve](distinct agg) add check of hash table to decide whether emplace value (#32063)
add 52eebc3ad6c [doc](ldap) opt the doc format (#31956)
add f3591247745 [Feat](nereids) support column default value current_date (#32268)
add 84572d7cb84 [feature][insert]add hive table sink thrift (#32274)
add 7828b7585b7 [fix](nereids) fix bug when grouping has same grouping set (#32235)
add 05c319b2baa [Fix](Variant) support view for accessing variant subcolumns and temp… (#32225)
add 758cb4385fb [Fix](segment write) handle variant bloom filter in segment writer (#32011)
add de0656586ff [RuntimeFilter](exec) support min max runtime filter and do refactor (#32210)
add ad4528826d4 [fix](cloud) ignore some case in cloud mode (#32261)
add ddddf02ad60 [feature](merge-cloud) enable merge on write by default (#32175)
add 140eb7f49c6 [fix](nereids)AssertNumRow node's output should be nullable (#32136)
add 01f924b2d70 [fix](function)revert function REPEAT nullable mode #32226
add a97af1a08c1 [bugfix](profile) support multi execution profile for brokerload (#32280)
add 8fb35130046 [fix](Nereids): don't pushdown project when project contains both side of join (#32214)
add 667ae3e5187 [fix](retry) Set query encounter rpc exception default retry times to 3 (#28555)
add 8c592461c03 [Chore](FE)Remove unused components (#32295)
add 37c6497d821 [Fix](nereids) remove duplicate expr in grouping set (#32290)
add b1d63402d6c Add QUEUE_START_TIME/QUEUE_END_TIME/QUERY_STATUS column for active_queries (#32259)
add fb23add819b [fix](s3 client)add default ca cert list for s3 client to avoid problem:'curlCode:77' (#32285)
add 1db78fe52c9 [fix](index tool)Fix index_tool build error (#32289)
add 588715496fb [fix](jni) remove 'push_down_predicates' and fix BE crash with decimal predicate (#32253)
add e4f0c6032fa [fix](compatibility) should enable windown funnel mode from 2.0 (#32284)
add f0f83e0c1c9 [docs](docs) Fix invalid link and typo of Master Branch (#32213)
add 277b3cf4379 [pipelineX](exchange) Make exchange buffer size configurable (#32201)
add 7be5a5c355d [Optimize](inverted index) optimize inverted index bitmap copy (#32279)
add 6d45d88273b [enhance][fix] add ccr downstreamurl for ccr (#32325)
add 241d14cdb7a [Enhancement](inverted index) make compiler happy (#32332)
add 3e1574db71a [fix](datetime) fix datetime rounding on BE (#32075)
add bb409748679 fix load channel may memory leak (#32277)
add 53bbbe6d782 [feature](Cloud) Support drop storage vault in Meta Service (#32228)
add 0345a592237 [refactor](chore) replace HashMapWithStackMemory with std::unordered_map (#32309)
add e9aae7affa0 [feature](hive)add 'HmsCommiter' to support inserting data into hive table (#32283)
add da72a61bed2 [Feature-WIP](multi-catalog) Add Hive sink on BE side. (#32306)
add 9e3a75b7484 [feature](cloud) multi cloud cluster (#31749)
add 1fd6085ac1c [fix](cloud) only set vault name if it's not null nor empty and check it in ms (#32351)
add f423bff93ae [doc](ranger)change path of access_controller.class (#32138)
add 601887ac2fa [improve](cache) Refactor block file cache and add more ut case (#30039)
add 323fc822f0b [behavior change](output) change float output format (#32049)
add aae5e6fc525 [opt](conf) modify the default fe be config file (#32269)
add 5637f065b9a [fix](profile) avoid update profile in deconstructor (#32131)
add 57d9723715b [chore](build) delete palo_be soft link (#32353)
add 546f6896bf9 [feature](proxy-protocol) Support proxy protocol v1 (#32338)
add 6a36d613bef [opt](file-meta-cache) reduce file meta cache size and disable cache for some cases (#32340)
add 0d55c6e6474 [bug](inverted index) fix npe of InvertedIndexStorageFormat in table property (#32357)
add ccb122ca257 [fix](grouping sets) fix grouping sets have multiple empty sets (#32317)
add 95fcb5603c5 [feature-wip](ranger)support datamask and row filter (#32137)
add 79a8620a130 [enhancement](nereids)simplify OneRowRelation scalar subquery (#32276)
add 0deac158375 [feature](insert)implement hive table sink plan (#31765)
add e32eef5f0be [Fix](Nereids) fix leading syntax problems and data mismatched problem (#32286)
add c2e155e30f9 [enhance](auth)row policy support catalog and match name instead id (#32310)
add 06801d5518b [Improve](spark-load)update spark version for spark load to resolve cve problem (#30368)
add 653925ceb8f [opt](jdbc catalog) Refine the jdbc_connector close logic and actively clear the jvm occupied by jdbcexecutor (#32300)
add de0497b7ef3 [opt](jdbc catalog) filter jdbc datasource internal database (#32294)
add 6881cc1b65f [enhancement](dict) check valid of offset in page (#32349)
add 4f9976eb07a [opt](routine-load) enhance auto resume to keep routine load stable (#32239)
add dcbd5035cd8 [docs](MoveRepo) Update .asf.yaml (#32391)
add 200df314753 [docs](MoveRepo) Delete Docs in apache/doris (#32384)
add d832336ae12 [bug](fold) fix fold constant rule can't handle variable expr (#32313)
add d542248e1b5 [regression-test]( fix case ) fix case that using same table in one db with another case (#32380)
new 1bc42cd821a Collect high priority columns. (#31235)
new a4cba9f4b7e High priority queue and map. (#31509)
new 6fc7aa15ad5 Support column level health value. (#31794)
new fe58392bd47 Support follower sync query columns to master. (#31859)
new 8631bcafdc3 Support show auto analyze pending jobs. (#31926)
new c28bdc7fd75 Check column health value earlier, show job priority. (#32064)
new f74a4d07d83 support window (#32094)
new 7bcec7100f9 Refactor. (#32273)
new 6ad12223884 refactor2 (#32278)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (3fb4b214172)
\
N -- N -- N refs/heads/high-priority-column (6ad12223884)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.asf.yaml | 2 -
be/src/agent/be_exec_version_manager.h | 3 +-
be/src/agent/cgroup_cpu_ctl.cpp | 2 +-
be/src/agent/workload_group_listener.cpp | 35 +-
be/src/cloud/cloud_rowset_writer.cpp | 10 +-
be/src/cloud/cloud_storage_engine.cpp | 8 +-
be/src/cloud/cloud_tablet.cpp | 4 +-
be/src/cloud/cloud_vertical_rowset_writer.cpp | 71 -
be/src/cloud/cloud_vertical_rowset_writer.h | 60 -
be/src/common/config.cpp | 47 +-
be/src/common/config.h | 26 +-
be/src/common/daemon.cpp | 2 +-
be/src/exec/data_sink.cpp | 16 +
be/src/exec/line_reader.h | 3 +-
be/src/exec/schema_scanner.cpp | 3 +
.../schema_active_queries_scanner.cpp | 25 +-
.../schema_workload_groups_scanner.cpp | 166 +
.../schema_workload_groups_scanner.h | 52 +
be/src/exec/tablet_info.cpp | 4 +-
be/src/exprs/bloom_filter_func.h | 5 +-
be/src/exprs/hybrid_set.h | 30 +-
be/src/exprs/minmax_predicate.h | 14 +-
be/src/exprs/runtime_filter.cpp | 77 +-
be/src/gutil/strings/numbers.cc | 14 +-
be/src/http/action/clear_file_cache_action.cpp | 40 +
be/src/http/action/clear_file_cache_action.h | 32 +
be/src/http/action/file_cache_action.cpp | 2 +-
be/src/index-tools/index_tool.cpp | 242 +-
be/src/io/cache/block/block_file_cache.cpp | 229 -
be/src/io/cache/block/block_file_cache.h | 342 -
be/src/io/cache/block/block_file_cache_factory.cpp | 116 -
be/src/io/cache/block/block_file_cache_factory.h | 71 -
be/src/io/cache/block/block_file_cache_fwd.h | 36 -
be/src/io/cache/block/block_file_cache_profile.cpp | 142 -
be/src/io/cache/block/block_file_cache_profile.h | 152 -
be/src/io/cache/block/block_file_cache_settings.h | 43 -
be/src/io/cache/block/block_file_segment.cpp | 361 -
be/src/io/cache/block/block_file_segment.h | 213 -
be/src/io/cache/block/block_lru_file_cache.cpp | 1144 ---
be/src/io/cache/block/block_lru_file_cache.h | 231 -
.../io/cache/block/cached_remote_file_reader.cpp | 285 -
be/src/io/cache/block/cached_remote_file_reader.h | 85 -
be/src/io/cache/block_file_cache.cpp | 1507 ++++
be/src/io/cache/block_file_cache.h | 420 +
be/src/io/cache/block_file_cache_factory.cpp | 131 +
be/src/io/cache/block_file_cache_factory.h | 75 +
be/src/io/cache/block_file_cache_profile.cpp | 76 +
be/src/io/cache/block_file_cache_profile.h | 127 +
be/src/io/cache/cached_remote_file_reader.cpp | 267 +
be/src/io/cache/cached_remote_file_reader.h | 83 +
be/src/io/cache/file_block.cpp | 276 +
be/src/io/cache/file_block.h | 175 +
be/src/io/cache/file_cache_common.cpp | 58 +
be/src/io/cache/file_cache_common.h | 117 +
be/src/io/cache/file_cache_storage.h | 49 +
be/src/io/cache/fs_file_cache_storage.cpp | 578 ++
be/src/io/cache/fs_file_cache_storage.h | 97 +
be/src/io/fs/benchmark/base_benchmark.h | 2 +-
be/src/io/fs/buffered_reader.cpp | 24 +-
be/src/io/fs/buffered_reader.h | 47 +-
be/src/io/fs/err_utils.cpp | 8 +-
be/src/io/fs/file_reader.h | 3 +-
be/src/io/fs/file_writer.h | 4 +-
be/src/io/fs/hdfs_file_reader.cpp | 74 +-
be/src/io/fs/hdfs_file_reader.h | 2 +
be/src/io/fs/local_file_reader.cpp | 3 +
be/src/io/fs/local_file_system.cpp | 7 +
be/src/io/fs/local_file_writer.cpp | 6 +-
be/src/io/fs/multi_table_pipe.cpp | 2 +
be/src/io/fs/remote_file_system.cpp | 2 +-
be/src/io/fs/s3_file_bufferpool.cpp | 35 +-
be/src/io/fs/s3_file_bufferpool.h | 8 +-
be/src/io/fs/s3_file_writer.cpp | 28 +-
be/src/io/fs/s3_file_writer.h | 7 +-
be/src/io/io_common.h | 2 +-
be/src/olap/base_tablet.cpp | 2 +-
be/src/olap/compaction.cpp | 153 +-
be/src/olap/comparison_predicate.h | 8 +-
be/src/olap/delta_writer.cpp | 27 +-
be/src/olap/in_list_predicate.h | 6 +-
be/src/olap/match_predicate.cpp | 8 +-
be/src/olap/options.cpp | 23 +-
be/src/olap/options.h | 8 +-
be/src/olap/rowset/beta_rowset.cpp | 247 +-
be/src/olap/rowset/beta_rowset.h | 2 +-
be/src/olap/rowset/beta_rowset_writer.cpp | 142 +-
be/src/olap/rowset/beta_rowset_writer.h | 31 +-
be/src/olap/rowset/beta_rowset_writer_v2.cpp | 3 +-
be/src/olap/rowset/beta_rowset_writer_v2.h | 6 +-
be/src/olap/rowset/rowset.cpp | 12 -
be/src/olap/rowset/rowset.h | 2 +-
be/src/olap/rowset/rowset_factory.cpp | 5 +-
be/src/olap/rowset/rowset_meta.cpp | 34 +
be/src/olap/rowset/rowset_meta.h | 17 +-
be/src/olap/rowset/rowset_writer_context.h | 2 +-
be/src/olap/rowset/segcompaction.cpp | 17 +-
be/src/olap/rowset/segment_creator.cpp | 126 +-
be/src/olap/rowset/segment_creator.h | 21 +-
be/src/olap/rowset/segment_v2/binary_plain_page.h | 14 +-
be/src/olap/rowset/segment_v2/column_reader.cpp | 28 +-
be/src/olap/rowset/segment_v2/column_reader.h | 13 +-
be/src/olap/rowset/segment_v2/column_writer.cpp | 44 +-
be/src/olap/rowset/segment_v2/column_writer.h | 2 +
.../rowset/segment_v2/inverted_index_cache.cpp | 11 -
.../olap/rowset/segment_v2/inverted_index_cache.h | 28 -
.../segment_v2/inverted_index_compaction.cpp | 46 +-
.../rowset/segment_v2/inverted_index_compaction.h | 14 +-
.../inverted_index_compound_directory.cpp | 940 ---
.../segment_v2/inverted_index_compound_directory.h | 247 -
.../segment_v2/inverted_index_compound_reader.cpp | 41 +-
.../segment_v2/inverted_index_compound_reader.h | 53 +-
.../olap/rowset/segment_v2/inverted_index_desc.cpp | 10 +-
.../olap/rowset/segment_v2/inverted_index_desc.h | 4 +
.../segment_v2/inverted_index_file_reader.cpp | 281 +
.../rowset/segment_v2/inverted_index_file_reader.h | 94 +
.../segment_v2/inverted_index_file_writer.cpp | 414 +
.../rowset/segment_v2/inverted_index_file_writer.h | 97 +
.../segment_v2/inverted_index_fs_directory.cpp | 751 ++
.../segment_v2/inverted_index_fs_directory.h | 232 +
.../rowset/segment_v2/inverted_index_reader.cpp | 147 +-
.../olap/rowset/segment_v2/inverted_index_reader.h | 75 +-
.../rowset/segment_v2/inverted_index_searcher.cpp | 13 +-
.../rowset/segment_v2/inverted_index_searcher.h | 17 +-
.../rowset/segment_v2/inverted_index_writer.cpp | 224 +-
.../olap/rowset/segment_v2/inverted_index_writer.h | 5 +-
be/src/olap/rowset/segment_v2/segment.cpp | 28 +-
be/src/olap/rowset/segment_v2/segment.h | 5 +
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 35 +-
be/src/olap/rowset/segment_v2/segment_writer.cpp | 30 +-
be/src/olap/rowset/segment_v2/segment_writer.h | 2 +
.../rowset/segment_v2/vertical_segment_writer.cpp | 22 +-
.../rowset/segment_v2/vertical_segment_writer.h | 2 +
be/src/olap/rowset/vertical_beta_rowset_writer.cpp | 185 +-
be/src/olap/rowset/vertical_beta_rowset_writer.h | 24 +-
.../rowset/vertical_beta_rowset_writer_helper.cpp | 244 -
.../rowset/vertical_beta_rowset_writer_helper.h | 83 -
be/src/olap/tablet_meta.cpp | 17 +-
be/src/olap/tablet_meta.h | 4 +-
be/src/olap/tablet_reader.cpp | 7 +-
be/src/olap/tablet_reader.h | 2 +-
be/src/olap/tablet_schema.cpp | 11 +-
be/src/olap/tablet_schema.h | 12 +
be/src/olap/task/index_builder.cpp | 252 +-
be/src/olap/task/index_builder.h | 9 +-
be/src/pipeline/exec/assert_num_rows_operator.cpp | 47 +-
be/src/pipeline/exec/assert_num_rows_operator.h | 1 +
.../distinct_streaming_aggregation_operator.cpp | 130 +-
.../exec/distinct_streaming_aggregation_operator.h | 4 +
...istinct_streaming_aggregation_sink_operator.cpp | 10 +-
...tinct_streaming_aggregation_source_operator.cpp | 3 +-
...istinct_streaming_aggregation_source_operator.h | 1 -
be/src/pipeline/exec/exchange_sink_buffer.cpp | 6 +-
be/src/pipeline/exec/exchange_sink_buffer.h | 1 -
be/src/pipeline/exec/exchange_sink_operator.cpp | 73 +-
be/src/pipeline/exec/exchange_sink_operator.h | 28 +
be/src/pipeline/exec/hive_table_sink_operator.cpp | 50 +
be/src/pipeline/exec/hive_table_sink_operator.h | 115 +
be/src/pipeline/exec/repeat_operator.cpp | 29 +-
be/src/pipeline/exec/repeat_operator.h | 5 +-
be/src/pipeline/exec/scan_operator.cpp | 3 +-
be/src/pipeline/pipeline_fragment_context.cpp | 11 +-
be/src/pipeline/pipeline_task.h | 2 +-
be/src/pipeline/pipeline_tracing.cpp | 10 +-
be/src/pipeline/pipeline_tracing.h | 4 +-
be/src/pipeline/pipeline_x/operator.cpp | 3 +
.../pipeline_x/pipeline_x_fragment_context.cpp | 9 +
be/src/pipeline/pipeline_x/pipeline_x_task.h | 2 +-
be/src/pipeline/task_queue.h | 2 +-
be/src/pipeline/task_scheduler.h | 2 +-
be/src/runtime/exec_env.h | 12 +-
be/src/runtime/exec_env_init.cpp | 74 +-
be/src/runtime/fragment_mgr.cpp | 34 +-
be/src/runtime/memory/lru_cache_policy.h | 7 +-
be/src/runtime/memory/mem_tracker_limiter.cpp | 8 +-
be/src/runtime/memory/mem_tracker_limiter.h | 14 +-
be/src/runtime/query_context.cpp | 30 +-
be/src/runtime/query_context.h | 6 +-
.../routine_load/routine_load_task_executor.cpp | 6 +
be/src/runtime/runtime_predicate.h | 8 +-
be/src/runtime/runtime_state.cpp | 12 +-
be/src/runtime/runtime_state.h | 4 +
be/src/runtime/stream_load/stream_load_context.h | 4 +
be/src/runtime/task_group/task_group.cpp | 439 --
be/src/runtime/task_group/task_group.h | 192 -
be/src/runtime/task_group/task_group_manager.cpp | 143 -
be/src/runtime/task_group/task_group_manager.h | 69 -
be/src/runtime/types.cpp | 4 +-
be/src/runtime/workload_group/workload_group.cpp | 437 ++
be/src/runtime/workload_group/workload_group.h | 189 +
.../workload_group/workload_group_manager.cpp | 144 +
.../workload_group/workload_group_manager.h | 66 +
be/src/service/doris_main.cpp | 2 +-
be/src/service/http_service.cpp | 6 +-
be/src/service/internal_service.cpp | 32 +-
be/src/util/indexed_priority_queue.hpp | 182 +
be/src/util/jni-util.cpp | 4 +-
be/src/util/jni-util.h | 11 +-
be/src/util/mem_info.cpp | 46 +-
be/src/util/obj_lru_cache.cpp | 7 +
be/src/util/obj_lru_cache.h | 2 +
be/src/util/profile_collector.h | 45 +
be/src/util/s3_util.cpp | 23 +
be/src/util/s3_util.h | 2 +
.../aggregate_function_java_udaf.h | 2 +-
.../aggregate_function_window_funnel.h | 4 +-
be/src/vec/columns/subcolumn_tree.h | 18 +-
be/src/vec/core/block.cpp | 2 +-
be/src/vec/data_types/data_type_ipv4.h | 3 +
be/src/vec/data_types/data_type_ipv6.h | 3 +
be/src/vec/data_types/data_type_time_v2.cpp | 4 +-
be/src/vec/data_types/data_type_time_v2.h | 5 +-
be/src/vec/exec/distinct_vaggregation_node.cpp | 41 +-
be/src/vec/exec/distinct_vaggregation_node.h | 4 +-
.../file_reader/new_plain_text_line_reader.cpp | 7 +
.../file_reader/new_plain_text_line_reader.h | 3 +
be/src/vec/exec/format/generic_reader.h | 3 +-
be/src/vec/exec/format/jni_reader.h | 7 +
be/src/vec/exec/format/json/new_json_reader.cpp | 9 +
be/src/vec/exec/format/json/new_json_reader.h | 3 +
be/src/vec/exec/format/orc/vorc_reader.cpp | 19 +-
be/src/vec/exec/format/orc/vorc_reader.h | 9 +-
.../exec/format/parquet/vparquet_group_reader.cpp | 3 +-
.../exec/format/parquet/vparquet_group_reader.h | 9 +-
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 82 +-
be/src/vec/exec/format/parquet/vparquet_reader.h | 5 +
be/src/vec/exec/format/table/table_format_reader.h | 7 +
be/src/vec/exec/jni_connector.cpp | 78 +-
be/src/vec/exec/jni_connector.h | 9 +-
be/src/vec/exec/scan/new_olap_scanner.cpp | 6 +-
be/src/vec/exec/scan/new_olap_scanner.h | 2 +-
be/src/vec/exec/scan/scanner_context.h | 5 +-
be/src/vec/exec/scan/vfile_scanner.cpp | 28 +-
be/src/vec/exec/scan/vfile_scanner.h | 11 +
be/src/vec/exec/scan/vmeta_scanner.cpp | 42 -
be/src/vec/exec/scan/vscan_node.cpp | 2 +-
be/src/vec/exec/scan/vscanner.cpp | 2 +-
be/src/vec/exec/scan/vscanner.h | 4 +-
be/src/vec/exec/skewed_partition_rebalancer.cpp | 302 +
be/src/vec/exec/skewed_partition_rebalancer.h | 132 +
be/src/vec/exec/vaggregation_node.h | 6 +-
be/src/vec/exec/vassert_num_rows_node.cpp | 47 +-
be/src/vec/exec/vassert_num_rows_node.h | 1 +
be/src/vec/exec/vjdbc_connector.cpp | 4 +-
be/src/vec/exec/vrepeat_node.cpp | 24 +-
be/src/vec/exec/vrepeat_node.h | 3 +
be/src/vec/exprs/vdirect_in_predicate.h | 37 +-
be/src/vec/exprs/vectorized_fn_call.cpp | 29 +-
be/src/vec/exprs/vectorized_fn_call.h | 7 +
be/src/vec/exprs/vexpr.cpp | 1 +
be/src/vec/exprs/vexpr.h | 7 +
be/src/vec/exprs/vruntimefilter_wrapper.cpp | 86 +-
be/src/vec/exprs/vruntimefilter_wrapper.h | 3 +-
be/src/vec/functions/function_json.cpp | 3 +-
be/src/vec/functions/function_jsonb.cpp | 5 +-
be/src/vec/functions/function_string.cpp | 1 -
be/src/vec/functions/function_string.h | 97 -
be/src/vec/functions/in.h | 16 +-
be/src/vec/runtime/ipv4_value.h | 2 +-
be/src/vec/runtime/vdatetime_value.cpp | 84 +-
be/src/vec/runtime/vorc_transformer.cpp | 23 +-
be/src/vec/runtime/vorc_transformer.h | 6 +-
.../sink/scale_writer_partitioning_exchanger.hpp | 92 +
be/src/vec/sink/vhive_table_sink.cpp | 48 +
be/src/vec/sink/vhive_table_sink.h | 52 +
be/src/vec/sink/writer/vhive_partition_writer.cpp | 282 +
be/src/vec/sink/writer/vhive_partition_writer.h | 99 +
be/src/vec/sink/writer/vhive_table_writer.cpp | 432 ++
be/src/vec/sink/writer/vhive_table_writer.h | 74 +
be/src/vec/sink/writer/vhive_utils.cpp | 78 +
be/src/vec/sink/writer/vhive_utils.h | 45 +
be/src/vec/sink/writer/vtablet_writer.cpp | 2 +-
be/test/gutil/strings/numbers_test.cpp | 2 +-
be/test/io/cache/block_file_cache_test.cpp | 3646 +++++++++
be/test/io/cache/file_block_cache_test.cpp | 1040 ---
be/test/util/indexed_priority_queue_test.cpp | 104 +
be/test/vec/data_types/datetime_round_test.cpp | 421 +
be/test/vec/data_types/from_string_test.cpp | 15 +-
.../vec/exec/skewed_partition_rebalancer_test.cpp | 318 +
be/test/vec/exec/vhive_utils_test.cpp | 70 +
be/test/vec/exprs/vexpr_test.cpp | 11 +-
build.sh | 6 -
cloud/src/meta-service/meta_service.cpp | 31 +
cloud/src/meta-service/meta_service_resource.cpp | 53 +-
cloud/test/meta_service_test.cpp | 248 +-
conf/be.conf | 16 +-
conf/fe.conf | 18 +-
.../community/design/doris_storage_optimization.md | 235 -
docs/en/community/design/grouping_sets_design.md | 501 --
docs/en/community/design/metadata-design.md | 127 -
docs/en/community/design/spark_load.md | 245 -
docs/en/community/developer-guide/be-clion-dev.md | 95 -
docs/en/community/developer-guide/be-vscode-dev.md | 341 -
.../en/community/developer-guide/benchmark-tool.md | 151 -
.../developer-guide/bitmap-hll-file-format.md | 64 -
.../developer-guide/cpp-diagnostic-code.md | 54 -
.../community/developer-guide/cpp-format-code.md | 97 -
docs/en/community/developer-guide/debug-tool.md | 512 --
docs/en/community/developer-guide/docker-dev.md | 165 -
.../en/community/developer-guide/fe-eclipse-dev.md | 226 -
docs/en/community/developer-guide/fe-idea-dev.md | 217 -
docs/en/community/developer-guide/fe-vscode-dev.md | 84 -
docs/en/community/developer-guide/github-checks.md | 31 -
.../developer-guide/how-to-share-blogs.md | 39 -
.../community/developer-guide/java-format-code.md | 99 -
.../developer-guide/mac-dev/be-clion-dev.md | 78 -
.../developer-guide/mac-dev/dev-prepare.md | 163 -
.../developer-guide/mac-dev/fe-idea-dev.md | 80 -
.../developer-guide/regression-testing.md | 864 ---
docs/en/community/doris-future.md | 41 -
docs/en/community/feedback.md | 48 -
.../commit-format-specification.md | 164 -
.../community/how-to-contribute/contribute-doc.md | 496 --
.../how-to-contribute/contributor-guide.md | 87 -
.../how-to-contribute/how-to-be-a-committer.md | 37 -
.../how-to-contribute/how-to-contribute.md | 85 -
.../en/community/how-to-contribute/pull-request.md | 260 -
docs/en/community/join-community.md | 117 -
.../release-and-verify/release-complete.md | 190 -
.../release-and-verify/release-doris-connectors.md | 207 -
.../release-and-verify/release-doris-core.md | 324 -
.../release-and-verify/release-doris-sdk.md | 192 -
.../release-and-verify/release-doris-shade.md | 192 -
.../release-and-verify/release-prepare.md | 348 -
.../community/release-and-verify/release-verify.md | 132 -
docs/en/community/security.md | 38 -
docs/en/community/subscribe-mail-list.md | 69 -
docs/en/community/team.md | 112 -
docs/en/docs/admin-manual/certificate.md | 77 -
.../cluster-management/elastic-expansion.md | 154 -
.../docs/admin-manual/cluster-management/fqdn.md | 103 -
.../cluster-management/load-balancing.md | 640 --
.../admin-manual/cluster-management/upgrade.md | 320 -
docs/en/docs/admin-manual/config/be-config.md | 1507 ----
docs/en/docs/admin-manual/config/config-dir.md | 49 -
.../docs/admin-manual/config/fe-config-template.md | 141 -
docs/en/docs/admin-manual/config/fe-config.md | 2776 -------
docs/en/docs/admin-manual/config/user-property.md | 73 -
docs/en/docs/admin-manual/data-admin/backup.md | 250 -
.../docs/admin-manual/data-admin/data-recovery.md | 54 -
.../docs/admin-manual/data-admin/delete-recover.md | 192 -
docs/en/docs/admin-manual/data-admin/restore.md | 193 -
docs/en/docs/admin-manual/fe-certificate.md | 46 -
.../http-actions/be/check-rpc-channel.md | 71 -
.../http-actions/be/check-tablet-segment.md | 72 -
.../docs/admin-manual/http-actions/be/checksum.md | 64 -
.../admin-manual/http-actions/be/compaction-run.md | 133 -
.../http-actions/be/compaction-status.md | 112 -
.../en/docs/admin-manual/http-actions/be/config.md | 107 -
.../docs/admin-manual/http-actions/be/download.md | 59 -
.../en/docs/admin-manual/http-actions/be/health.md | 57 -
docs/en/docs/admin-manual/http-actions/be/meta.md | 73 -
.../docs/admin-manual/http-actions/be/metrics.md | 67 -
.../admin-manual/http-actions/be/pad-rowset.md | 68 -
.../http-actions/be/reset-rpc-channel.md | 67 -
.../docs/admin-manual/http-actions/be/snapshot.md | 62 -
.../http-actions/be/tablet-distribution.md | 92 -
.../admin-manual/http-actions/be/tablet-info.md | 78 -
.../http-actions/be/tablet-migration.md | 111 -
.../admin-manual/http-actions/be/tablet-reload.md | 65 -
.../admin-manual/http-actions/be/tablet-restore.md | 65 -
.../admin-manual/http-actions/be/version-info.md | 75 -
.../http-actions/fe/backends-action.md | 70 -
.../http-actions/fe/bootstrap-action.md | 124 -
.../http-actions/fe/cancel-load-action.md | 102 -
.../http-actions/fe/check-decommission-action.md | 84 -
.../http-actions/fe/check-storage-type-action.md | 84 -
.../admin-manual/http-actions/fe/cluster-action.md | 89 -
.../http-actions/fe/colocate-meta-action.md | 59 -
.../admin-manual/http-actions/fe/config-action.md | 85 -
.../http-actions/fe/connection-action.md | 82 -
.../http-actions/fe/debug-point-action.md | 340 -
.../http-actions/fe/extra-basepath-action.md | 58 -
.../http-actions/fe/fe-version-info-action.md | 101 -
.../http-actions/fe/get-ddl-stmt-action.md | 92 -
.../http-actions/fe/get-load-info-action.md | 95 -
.../admin-manual/http-actions/fe/get-load-state.md | 94 -
.../http-actions/fe/get-log-file-action.md | 113 -
.../admin-manual/http-actions/fe/get-small-file.md | 102 -
.../docs/admin-manual/http-actions/fe/ha-action.md | 95 -
.../http-actions/fe/hardware-info-action.md | 79 -
.../admin-manual/http-actions/fe/health-action.md | 61 -
.../admin-manual/http-actions/fe/help-action.md | 61 -
.../admin-manual/http-actions/fe/import-action.md | 53 -
.../admin-manual/http-actions/fe/log-action.md | 98 -
.../admin-manual/http-actions/fe/login-action.md | 70 -
.../admin-manual/http-actions/fe/logout-action.md | 59 -
.../admin-manual/http-actions/fe/meta-action.md | 64 -
.../http-actions/fe/meta-info-action-V2.md | 72 -
.../http-actions/fe/meta-info-action.md | 232 -
.../http-actions/fe/meta-replay-state-action.md | 61 -
.../admin-manual/http-actions/fe/metrics-action.md | 55 -
.../admin-manual/http-actions/fe/node-action.md | 622 --
.../admin-manual/http-actions/fe/profile-action.md | 189 -
.../http-actions/fe/query-detail-action.md | 116 -
.../fe/query-profile-action-controller.md | 108 -
.../http-actions/fe/query-profile-action.md | 491 --
.../http-actions/fe/query-schema-action.md | 108 -
.../http-actions/fe/query-stats-action.md | 99 -
.../http-actions/fe/row-count-action.md | 84 -
.../admin-manual/http-actions/fe/session-action.md | 124 -
.../http-actions/fe/set-config-action.md | 150 -
.../http-actions/fe/show-data-action.md | 111 -
.../http-actions/fe/show-meta-info-action.md | 138 -
.../http-actions/fe/show-proc-action.md | 104 -
.../http-actions/fe/show-runtime-info-action.md | 84 -
.../http-actions/fe/show-table-data-action.md | 163 -
.../http-actions/fe/statement-execution-action.md | 102 -
.../http-actions/fe/statistic-action.md | 58 -
.../admin-manual/http-actions/fe/system-action.md | 93 -
.../http-actions/fe/table-query-plan-action.md | 113 -
.../http-actions/fe/table-row-count-action.md | 88 -
.../http-actions/fe/table-schema-action.md | 155 -
.../admin-manual/http-actions/fe/upload-action.md | 159 -
.../maint-monitor/automatic-service-start.md | 348 -
.../maint-monitor/be-olap-error-code.md | 255 -
.../admin-manual/maint-monitor/disk-capacity.md | 167 -
.../admin-manual/maint-monitor/doris-error-code.md | 179 -
.../memory-management/be-oom-analysis.md | 84 -
.../memory-limit-exceeded-analysis.md | 179 -
.../memory-management/memory-tracker.md | 98 -
.../maint-monitor/metadata-operation.md | 413 -
.../admin-manual/maint-monitor/monitor-alert.md | 308 -
.../maint-monitor/monitor-metrics/metrics.md | 339 -
.../admin-manual/maint-monitor/multi-tenant.md | 238 -
.../maint-monitor/tablet-local-debug.md | 224 -
.../admin-manual/maint-monitor/tablet-meta-tool.md | 147 -
.../maint-monitor/tablet-repair-and-balance.md | 781 --
docs/en/docs/admin-manual/multi-tenant.md | 271 -
docs/en/docs/admin-manual/privilege-ldap/ldap.md | 243 -
docs/en/docs/admin-manual/privilege-ldap/ranger.md | 249 -
.../admin-manual/privilege-ldap/user-privilege.md | 296 -
docs/en/docs/admin-manual/query-profile.md | 278 -
docs/en/docs/admin-manual/sql-interception.md | 99 -
docs/en/docs/admin-manual/system-table/rowsets.md | 82 -
docs/en/docs/admin-manual/workload-group.md | 223 -
docs/en/docs/advanced/alter-table/replace-table.md | 72 -
docs/en/docs/advanced/alter-table/schema-change.md | 287 -
docs/en/docs/advanced/auto-increment.md | 508 --
docs/en/docs/advanced/autobucket.md | 178 -
docs/en/docs/advanced/best-practice/compaction.md | 74 -
docs/en/docs/advanced/best-practice/debug-log.md | 121 -
.../docs/advanced/best-practice/import-analysis.md | 202 -
.../docs/advanced/best-practice/query-analysis.md | 676 --
docs/en/docs/advanced/broker.md | 275 -
.../docs/advanced/cache/partition-cache-manual.md | 170 -
docs/en/docs/advanced/cache/query-cache.md | 140 -
docs/en/docs/advanced/cache/sql-cache-manual.md | 78 -
docs/en/docs/advanced/cold-hot-separation.md | 209 -
docs/en/docs/advanced/compute-node.md | 121 -
docs/en/docs/advanced/is-being-synced.md | 65 -
docs/en/docs/advanced/lateral-view.md | 94 -
docs/en/docs/advanced/orthogonal-bitmap-manual.md | 202 -
docs/en/docs/advanced/orthogonal-hll-manual.md | 28 -
docs/en/docs/advanced/partition/auto-partition.md | 260 -
.../docs/advanced/partition/dynamic-partition.md | 562 --
.../advanced/partition/table-temp-partition.md | 288 -
docs/en/docs/advanced/resource.md | 152 -
docs/en/docs/advanced/small-file-mgr.md | 132 -
docs/en/docs/advanced/sql-mode.md | 76 -
docs/en/docs/advanced/time-zone.md | 183 -
docs/en/docs/advanced/user-defined-variables.md | 115 -
docs/en/docs/advanced/using-hll.md | 208 -
docs/en/docs/advanced/variables-template.md | 116 -
docs/en/docs/advanced/variables.md | 736 --
docs/en/docs/benchmark/ssb.md | 609 --
docs/en/docs/benchmark/tpch.md | 880 ---
docs/en/docs/data-operate/export/export-manual.md | 236 -
.../data-operate/export/export-with-mysql-dump.md | 41 -
docs/en/docs/data-operate/export/outfile.md | 168 -
.../import/import-advanced/min-load-replica-num.md | 101 -
.../import/import-scenes/external-storage-load.md | 193 -
.../import/import-scenes/external-table-load.md | 123 -
.../data-operate/import/import-scenes/jdbc-load.md | 163 -
.../import/import-scenes/kafka-load.md | 133 -
.../import/import-scenes/load-atomicity.md | 49 -
.../import/import-scenes/load-data-convert.md | 366 -
.../import/import-scenes/load-strict-mode.md | 180 -
.../import/import-scenes/local-file-load.md | 275 -
.../import/import-way/binlog-load-manual.md | 552 --
.../import/import-way/broker-load-manual.md | 445 --
.../import/import-way/group-commit-manual.md | 501 --
.../import/import-way/insert-into-manual.md | 264 -
.../import/import-way/load-json-format.md | 617 --
.../import/import-way/mysql-load-manual.md | 152 -
.../import/import-way/routine-load-manual.md | 606 --
.../import/import-way/s3-load-manual.md | 111 -
.../import/import-way/spark-load-manual.md | 806 --
.../import/import-way/stream-load-manual.md | 525 --
docs/en/docs/data-operate/import/load-manual.md | 100 -
.../update-delete/batch-delete-manual.md | 307 -
.../data-operate/update-delete/delete-manual.md | 161 -
.../data-operate/update-delete/partial-update.md | 231 -
.../update-delete/sequence-column-manual.md | 256 -
docs/en/docs/data-operate/update-delete/update.md | 132 -
docs/en/docs/data-table/advance-usage.md | 294 -
docs/en/docs/data-table/basic-usage.md | 716 --
docs/en/docs/data-table/best-practice.md | 182 -
docs/en/docs/data-table/data-model.md | 807 --
docs/en/docs/data-table/data-partition.md | 424 --
docs/en/docs/data-table/hit-the-rollup.md | 452 --
docs/en/docs/data-table/index/bloomfilter.md | 130 -
docs/en/docs/data-table/index/index-overview.md | 87 -
docs/en/docs/data-table/index/inverted-index.md | 566 --
.../data-table/index/ngram-bloomfilter-index.md | 82 -
docs/en/docs/ecosystem/audit-plugin.md | 219 -
docs/en/docs/ecosystem/beats.md | 280 -
.../ecosystem/bi-integration/apache-superset.md | 53 -
docs/en/docs/ecosystem/cloudcanal.md | 108 -
docs/en/docs/ecosystem/datax.md | 393 -
docs/en/docs/ecosystem/dbt-doris-adapter.md | 261 -
docs/en/docs/ecosystem/doris-kafka-connector.md | 192 -
docs/en/docs/ecosystem/doris-streamloader.md | 250 -
docs/en/docs/ecosystem/flink-doris-connector.md | 790 --
docs/en/docs/ecosystem/hive-bitmap-udf.md | 165 -
docs/en/docs/ecosystem/kyuubi.md | 115 -
docs/en/docs/ecosystem/logstash.md | 204 -
.../en/docs/ecosystem/plugin-development-manual.md | 314 -
docs/en/docs/ecosystem/seatunnel.md | 323 -
docs/en/docs/ecosystem/spark-doris-connector.md | 375 -
.../remote-user-defined-aggregation-function.md | 115 -
docs/en/docs/ecosystem/udf/contribute-udf.md | 124 -
.../ecosystem/udf/java-user-defined-function.md | 348 -
.../ecosystem/udf/remote-user-defined-function.md | 150 -
docs/en/docs/faq/data-faq.md | 182 -
docs/en/docs/faq/install-faq.md | 318 -
docs/en/docs/faq/sql-faq.md | 101 -
docs/en/docs/get-starting/quick-start.md | 432 --
docs/en/docs/get-starting/what-is-apache-doris.md | 92 -
.../install/construct-docker/build-docker-image.md | 297 -
.../install/construct-docker/run-docker-cluster.md | 188 -
docs/en/docs/install/k8s-deploy/debug-crash.md | 73 -
.../install/k8s-deploy/doris-cluster-upgrade.md | 286 -
.../k8s-deploy/expansion-and-contraction.md | 100 -
.../docs/install/k8s-deploy/helm-chart-deploy.md | 108 -
docs/en/docs/install/k8s-deploy/network.md | 377 -
docs/en/docs/install/k8s-deploy/operator-deploy.md | 121 -
.../docs/install/k8s-deploy/persistent-volume.md | 420 -
docs/en/docs/install/k8s-deploy/root-user-use.md | 63 -
.../docs/install/source-install/compilation-arm.md | 641 --
.../install/source-install/compilation-general.md | 284 -
.../docs/install/source-install/compilation-mac.md | 118 -
.../docs/install/source-install/compilation-win.md | 94 -
.../compilation-with-ldb-toolchain.md | 144 -
docs/en/docs/install/standard-deployment.md | 395 -
docs/en/docs/lakehouse/cloud-auth/cloud-auth.md | 61 -
docs/en/docs/lakehouse/compute-node.md | 145 -
docs/en/docs/lakehouse/external-statistics.md | 48 -
docs/en/docs/lakehouse/faq.md | 302 -
docs/en/docs/lakehouse/file.md | 201 -
docs/en/docs/lakehouse/filecache.md | 95 -
docs/en/docs/lakehouse/fs-benchmark-tool.md | 235 -
docs/en/docs/lakehouse/multi-catalog/dlf.md | 112 -
docs/en/docs/lakehouse/multi-catalog/es.md | 519 --
docs/en/docs/lakehouse/multi-catalog/hive.md | 580 --
docs/en/docs/lakehouse/multi-catalog/hudi.md | 109 -
docs/en/docs/lakehouse/multi-catalog/iceberg.md | 230 -
docs/en/docs/lakehouse/multi-catalog/jdbc.md | 1053 ---
.../en/docs/lakehouse/multi-catalog/max-compute.md | 89 -
.../docs/lakehouse/multi-catalog/multi-catalog.md | 324 -
docs/en/docs/lakehouse/multi-catalog/paimon.md | 195 -
docs/en/docs/lakehouse/sql-dialect.md | 129 -
.../async-materialized-view.md | 213 -
.../query-async-materialized-view.md | 471 --
.../hight-concurrent-point-query.md | 160 -
docs/en/docs/query-acceleration/hint/joinHint.md | 688 --
.../join-optimization/bucket-shuffle-join.md | 105 -
.../join-optimization/colocation-join.md | 430 --
.../join-optimization/doris-join-optimization.md | 215 -
.../join-optimization/runtime-filter.md | 287 -
.../docs/query-acceleration/materialized-view.md | 541 --
docs/en/docs/query-acceleration/nereids.md | 86 -
.../pipeline-execution-engine.md | 79 -
.../pipeline-x-execution-engine.md | 76 -
docs/en/docs/query-acceleration/statistics.md | 335 -
docs/en/docs/releasenotes/release-1.1.0.md | 379 -
docs/en/docs/releasenotes/release-1.1.1.md | 78 -
docs/en/docs/releasenotes/release-1.1.2.md | 84 -
docs/en/docs/releasenotes/release-1.1.3.md | 92 -
docs/en/docs/releasenotes/release-1.1.4.md | 72 -
docs/en/docs/releasenotes/release-1.1.5.md | 65 -
docs/en/docs/releasenotes/release-1.2.0.md | 563 --
docs/en/docs/releasenotes/release-1.2.1.md | 196 -
docs/en/docs/releasenotes/release-1.2.2.md | 254 -
docs/en/docs/releasenotes/release-1.2.3.md | 109 -
docs/en/docs/releasenotes/release-1.2.4.md | 81 -
docs/en/docs/releasenotes/release-1.2.5.md | 199 -
docs/en/docs/releasenotes/release-1.2.6.md | 135 -
docs/en/docs/releasenotes/release-1.2.7.md | 46 -
docs/en/docs/releasenotes/release-1.2.8.md | 47 -
docs/en/docs/releasenotes/release-2.0-Alpha.md | 135 -
docs/en/docs/releasenotes/release-2.0-Beta.md | 313 -
docs/en/docs/releasenotes/release-2.0.0.md | 235 -
docs/en/docs/releasenotes/release-2.0.1.md | 224 -
docs/en/docs/releasenotes/release-2.0.2.md | 157 -
docs/en/docs/releasenotes/release-2.0.3.md | 256 -
docs/en/docs/releasenotes/release-2.0.4.md | 67 -
docs/en/docs/releasenotes/release-2.0.5.md | 73 -
docs/en/docs/releasenotes/release-2.0.6.md | 59 -
docs/en/docs/releasenotes/release-2.1.0.md | 159 -
.../sql-functions/string-functions/repeat.md | 7 +
.../Create/CREATE-EXTERNAL-TABLE.md | 274 +
.../sql-reference/Data-Types/AGG_STATE.md | 4 +-
docs/sidebars.json | 1470 ----
docs/sidebarsCommunity.json | 80 -
.../community/design/doris_storage_optimization.md | 234 -
.../zh-CN/community/design/grouping_sets_design.md | 517 --
docs/zh-CN/community/design/metadata-design.md | 126 -
docs/zh-CN/community/design/spark_load.md | 212 -
.../community/developer-guide/be-clion-dev.md | 93 -
.../community/developer-guide/be-vscode-dev.md | 339 -
.../community/developer-guide/benchmark-tool.md | 150 -
.../developer-guide/bitmap-hll-file-format.md | 64 -
.../developer-guide/cpp-diagnostic-code.md | 54 -
.../community/developer-guide/cpp-format-code.md | 97 -
docs/zh-CN/community/developer-guide/debug-tool.md | 458 --
docs/zh-CN/community/developer-guide/docker-dev.md | 166 -
.../community/developer-guide/fe-eclipse-dev.md | 227 -
.../zh-CN/community/developer-guide/fe-idea-dev.md | 215 -
.../community/developer-guide/fe-vscode-dev.md | 84 -
.../community/developer-guide/github-checks.md | 31 -
.../developer-guide/how-to-share-blogs.md | 39 -
.../community/developer-guide/java-format-code.md | 96 -
.../developer-guide/mac-dev/be-clion-dev.md | 78 -
.../developer-guide/mac-dev/dev-prepare.md | 161 -
.../developer-guide/mac-dev/fe-idea-dev.md | 78 -
.../developer-guide/regression-testing.md | 825 --
docs/zh-CN/community/doris-future.md | 41 -
docs/zh-CN/community/feedback.md | 45 -
.../commit-format-specification.md | 162 -
.../community/how-to-contribute/contribute-doc.md | 493 --
.../how-to-contribute/contributor-guide.md | 92 -
.../how-to-contribute/how-to-be-a-committer.md | 37 -
.../how-to-contribute/how-to-contribute.md | 84 -
.../community/how-to-contribute/pull-request.md | 261 -
docs/zh-CN/community/join-community.md | 109 -
.../release-and-verify/release-complete.md | 187 -
.../release-and-verify/release-doris-connectors.md | 207 -
.../release-and-verify/release-doris-core.md | 323 -
.../release-and-verify/release-doris-sdk.md | 192 -
.../release-and-verify/release-doris-shade.md | 192 -
.../release-and-verify/release-prepare.md | 349 -
.../community/release-and-verify/release-verify.md | 136 -
docs/zh-CN/community/security.md | 34 -
docs/zh-CN/community/subscribe-mail-list.md | 69 -
docs/zh-CN/community/team.md | 114 -
docs/zh-CN/docs/admin-manual/certificate.md | 78 -
.../cluster-management/elastic-expansion.md | 149 -
.../docs/admin-manual/cluster-management/fqdn.md | 102 -
.../cluster-management/load-balancing.md | 639 --
.../admin-manual/cluster-management/upgrade.md | 321 -
docs/zh-CN/docs/admin-manual/config/be-config.md | 1537 ----
docs/zh-CN/docs/admin-manual/config/config-dir.md | 49 -
.../docs/admin-manual/config/fe-config-template.md | 139 -
docs/zh-CN/docs/admin-manual/config/fe-config.md | 2774 -------
.../docs/admin-manual/config/user-property.md | 73 -
docs/zh-CN/docs/admin-manual/data-admin/backup.md | 249 -
.../docs/admin-manual/data-admin/data-recovery.md | 54 -
.../docs/admin-manual/data-admin/delete-recover.md | 205 -
docs/zh-CN/docs/admin-manual/data-admin/restore.md | 194 -
docs/zh-CN/docs/admin-manual/fe-certificate.md | 46 -
.../http-actions/be/check-rpc-channel.md | 71 -
.../http-actions/be/check-tablet-segment.md | 74 -
.../docs/admin-manual/http-actions/be/checksum.md | 64 -
.../admin-manual/http-actions/be/compaction-run.md | 136 -
.../http-actions/be/compaction-status.md | 112 -
.../docs/admin-manual/http-actions/be/config.md | 105 -
.../docs/admin-manual/http-actions/be/download.md | 59 -
.../docs/admin-manual/http-actions/be/health.md | 55 -
.../docs/admin-manual/http-actions/be/meta.md | 73 -
.../docs/admin-manual/http-actions/be/metrics.md | 67 -
.../admin-manual/http-actions/be/pad-rowset.md | 68 -
.../http-actions/be/reset-rpc-channel.md | 68 -
.../docs/admin-manual/http-actions/be/snapshot.md | 62 -
.../http-actions/be/tablet-distribution.md | 92 -
.../admin-manual/http-actions/be/tablet-info.md | 78 -
.../http-actions/be/tablet-migration.md | 111 -
.../admin-manual/http-actions/be/tablet-reload.md | 65 -
.../admin-manual/http-actions/be/tablet-restore.md | 65 -
.../admin-manual/http-actions/be/version-info.md | 79 -
.../http-actions/fe/backends-action.md | 70 -
.../http-actions/fe/bootstrap-action.md | 128 -
.../http-actions/fe/cancel-load-action.md | 101 -
.../http-actions/fe/check-decommission-action.md | 84 -
.../http-actions/fe/check-storage-type-action.md | 84 -
.../admin-manual/http-actions/fe/cluster-action.md | 89 -
.../http-actions/fe/colocate-meta-action.md | 59 -
.../admin-manual/http-actions/fe/config-action.md | 85 -
.../http-actions/fe/connection-action.md | 82 -
.../http-actions/fe/debug-point-action.md | 326 -
.../http-actions/fe/extra-basepath-action.md | 58 -
.../http-actions/fe/fe-version-info-action.md | 101 -
.../http-actions/fe/get-ddl-stmt-action.md | 92 -
.../http-actions/fe/get-load-info-action.md | 95 -
.../admin-manual/http-actions/fe/get-load-state.md | 98 -
.../http-actions/fe/get-log-file-action.md | 112 -
.../admin-manual/http-actions/fe/get-small-file.md | 101 -
.../docs/admin-manual/http-actions/fe/ha-action.md | 95 -
.../http-actions/fe/hardware-info-action.md | 80 -
.../admin-manual/http-actions/fe/health-action.md | 61 -
.../admin-manual/http-actions/fe/help-action.md | 61 -
.../admin-manual/http-actions/fe/import-action.md | 53 -
.../admin-manual/http-actions/fe/log-action.md | 98 -
.../admin-manual/http-actions/fe/login-action.md | 70 -
.../admin-manual/http-actions/fe/logout-action.md | 59 -
.../admin-manual/http-actions/fe/meta-action.md | 60 -
.../http-actions/fe/meta-info-action-V2.md | 72 -
.../http-actions/fe/meta-info-action.md | 232 -
.../http-actions/fe/meta-replay-state-action.md | 61 -
.../admin-manual/http-actions/fe/metrics-action.md | 55 -
.../admin-manual/http-actions/fe/node-action.md | 622 --
.../admin-manual/http-actions/fe/profile-action.md | 189 -
.../http-actions/fe/query-detail-action.md | 116 -
.../fe/query-profile-action-controller.md | 108 -
.../http-actions/fe/query-profile-action.md | 492 --
.../http-actions/fe/query-schema-action.md | 107 -
.../http-actions/fe/query-stats-action.md | 100 -
.../http-actions/fe/row-count-action.md | 84 -
.../admin-manual/http-actions/fe/session-action.md | 124 -
.../http-actions/fe/set-config-action.md | 149 -
.../http-actions/fe/show-data-action.md | 111 -
.../http-actions/fe/show-meta-info-action.md | 138 -
.../http-actions/fe/show-proc-action.md | 104 -
.../http-actions/fe/show-runtime-info-action.md | 84 -
.../http-actions/fe/show-table-data-action.md | 163 -
.../http-actions/fe/statement-execution-action.md | 102 -
.../http-actions/fe/statistic-action.md | 58 -
.../admin-manual/http-actions/fe/system-action.md | 93 -
.../http-actions/fe/table-query-plan-action.md | 113 -
.../http-actions/fe/table-row-count-action.md | 88 -
.../http-actions/fe/table-schema-action.md | 155 -
.../admin-manual/http-actions/fe/upload-action.md | 159 -
.../maint-monitor/automatic-service-start.md | 336 -
.../maint-monitor/be-olap-error-code.md | 261 -
.../admin-manual/maint-monitor/disk-capacity.md | 161 -
.../admin-manual/maint-monitor/doris-error-code.md | 179 -
.../memory-management/be-oom-analysis.md | 84 -
.../memory-limit-exceeded-analysis.md | 179 -
.../memory-management/memory-tracker.md | 98 -
.../maint-monitor/metadata-operation.md | 416 -
.../admin-manual/maint-monitor/monitor-alert.md | 308 -
.../maint-monitor/monitor-metrics/metrics.md | 340 -
.../maint-monitor/tablet-local-debug.md | 224 -
.../admin-manual/maint-monitor/tablet-meta-tool.md | 146 -
.../maint-monitor/tablet-repair-and-balance.md | 783 --
docs/zh-CN/docs/admin-manual/multi-tenant.md | 270 -
.../zh-CN/docs/admin-manual/privilege-ldap/ldap.md | 261 -
.../docs/admin-manual/privilege-ldap/ranger.md | 249 -
.../admin-manual/privilege-ldap/user-privilege.md | 283 -
docs/zh-CN/docs/admin-manual/query-profile.md | 280 -
docs/zh-CN/docs/admin-manual/sql-interception.md | 98 -
.../docs/admin-manual/system-table/rowsets.md | 86 -
docs/zh-CN/docs/admin-manual/workload-group.md | 219 -
.../docs/advanced/alter-table/replace-table.md | 71 -
.../docs/advanced/alter-table/schema-change.md | 295 -
docs/zh-CN/docs/advanced/auto-increment.md | 505 --
docs/zh-CN/docs/advanced/autobucket.md | 179 -
.../docs/advanced/best-practice/compaction.md | 71 -
.../zh-CN/docs/advanced/best-practice/debug-log.md | 120 -
.../docs/advanced/best-practice/import-analysis.md | 201 -
.../docs/advanced/best-practice/query-analysis.md | 686 --
docs/zh-CN/docs/advanced/broker.md | 268 -
.../docs/advanced/cache/partition-cache-manual.md | 170 -
docs/zh-CN/docs/advanced/cache/query-cache.md | 140 -
docs/zh-CN/docs/advanced/cache/sql-cache-manual.md | 78 -
docs/zh-CN/docs/advanced/cold-hot-separation.md | 208 -
docs/zh-CN/docs/advanced/is-being-synced.md | 65 -
docs/zh-CN/docs/advanced/lateral-view.md | 94 -
.../docs/advanced/orthogonal-bitmap-manual.md | 205 -
.../docs/advanced/partition/auto-partition.md | 260 -
.../docs/advanced/partition/dynamic-partition.md | 553 --
.../advanced/partition/table-temp-partition.md | 286 -
docs/zh-CN/docs/advanced/resource.md | 147 -
docs/zh-CN/docs/advanced/small-file-mgr.md | 134 -
docs/zh-CN/docs/advanced/sql-mode.md | 82 -
docs/zh-CN/docs/advanced/time-zone.md | 179 -
docs/zh-CN/docs/advanced/user-defined-variables.md | 108 -
docs/zh-CN/docs/advanced/using-hll.md | 208 -
docs/zh-CN/docs/advanced/variables-template.md | 115 -
docs/zh-CN/docs/advanced/variables.md | 726 --
docs/zh-CN/docs/benchmark/ssb.md | 615 --
docs/zh-CN/docs/benchmark/tpch.md | 884 ---
.../docs/data-operate/export/export-manual.md | 236 -
.../data-operate/export/export-with-mysql-dump.md | 41 -
docs/zh-CN/docs/data-operate/export/outfile.md | 162 -
.../import/import-advanced/min-load-replica-num.md | 102 -
.../import/import-scenes/external-storage-load.md | 199 -
.../import/import-scenes/external-table-load.md | 130 -
.../data-operate/import/import-scenes/jdbc-load.md | 163 -
.../import/import-scenes/kafka-load.md | 134 -
.../import/import-scenes/load-atomicity.md | 49 -
.../import/import-scenes/load-data-convert.md | 366 -
.../import/import-scenes/load-strict-mode.md | 180 -
.../import/import-scenes/local-file-load.md | 276 -
.../import/import-way/binlog-load-manual.md | 537 --
.../import/import-way/broker-load-manual.md | 445 --
.../import/import-way/group-commit-manual.md | 499 --
.../import/import-way/insert-into-manual.md | 281 -
.../import/import-way/load-json-format.md | 618 --
.../import/import-way/mysql-load-manual.md | 155 -
.../import/import-way/routine-load-manual.md | 579 --
.../import/import-way/s3-load-manual.md | 111 -
.../import/import-way/spark-load-manual.md | 770 --
.../import/import-way/stream-load-manual.md | 550 --
docs/zh-CN/docs/data-operate/import/load-manual.md | 102 -
.../update-delete/batch-delete-manual.md | 327 -
.../data-operate/update-delete/delete-manual.md | 156 -
.../data-operate/update-delete/partial-update.md | 229 -
.../update-delete/sequence-column-manual.md | 276 -
.../docs/data-operate/update-delete/update.md | 130 -
docs/zh-CN/docs/data-table/basic-usage.md | 723 --
docs/zh-CN/docs/data-table/best-practice.md | 181 -
docs/zh-CN/docs/data-table/data-model.md | 807 --
docs/zh-CN/docs/data-table/data-partition.md | 432 --
docs/zh-CN/docs/data-table/hit-the-rollup.md | 452 --
docs/zh-CN/docs/data-table/index/bloomfilter.md | 127 -
docs/zh-CN/docs/data-table/index/index-overview.md | 86 -
docs/zh-CN/docs/data-table/index/inverted-index.md | 564 --
.../data-table/index/ngram-bloomfilter-index.md | 84 -
docs/zh-CN/docs/ecosystem/audit-plugin.md | 222 -
docs/zh-CN/docs/ecosystem/beats.md | 282 -
.../ecosystem/bi-integration/apache-superset.md | 58 -
docs/zh-CN/docs/ecosystem/cloudcanal.md | 114 -
docs/zh-CN/docs/ecosystem/datax.md | 393 -
docs/zh-CN/docs/ecosystem/dbt-doris-adapter.md | 260 -
docs/zh-CN/docs/ecosystem/doris-kafka-connector.md | 193 -
docs/zh-CN/docs/ecosystem/doris-streamloader.md | 249 -
docs/zh-CN/docs/ecosystem/flink-doris-connector.md | 790 --
docs/zh-CN/docs/ecosystem/hive-bitmap-udf.md | 174 -
docs/zh-CN/docs/ecosystem/kyuubi.md | 113 -
docs/zh-CN/docs/ecosystem/logstash.md | 204 -
.../docs/ecosystem/plugin-development-manual.md | 312 -
docs/zh-CN/docs/ecosystem/seatunnel.md | 325 -
docs/zh-CN/docs/ecosystem/spark-doris-connector.md | 380 -
.../remote-user-defined-aggregation-function.md | 216 -
docs/zh-CN/docs/ecosystem/udf/contribute-udf.md | 124 -
.../ecosystem/udf/java-user-defined-function.md | 348 -
.../ecosystem/udf/remote-user-defined-function.md | 154 -
docs/zh-CN/docs/faq/data-faq.md | 180 -
docs/zh-CN/docs/faq/install-faq.md | 316 -
docs/zh-CN/docs/faq/sql-faq.md | 91 -
docs/zh-CN/docs/get-starting/quick-start.md | 439 --
.../docs/get-starting/what-is-apache-doris.md | 98 -
.../install/construct-docker/build-docker-image.md | 303 -
.../install/construct-docker/run-docker-cluster.md | 190 -
docs/zh-CN/docs/install/k8s-deploy/debug-crash.md | 73 -
.../install/k8s-deploy/doris-cluster-upgrade.md | 291 -
.../k8s-deploy/expansion-and-contraction.md | 100 -
.../docs/install/k8s-deploy/helm-chart-deploy.md | 105 -
docs/zh-CN/docs/install/k8s-deploy/network.md | 387 -
.../docs/install/k8s-deploy/operator-deploy.md | 122 -
.../docs/install/k8s-deploy/persistent-volume.md | 421 -
.../zh-CN/docs/install/k8s-deploy/root-user-use.md | 63 -
.../docs/install/source-install/compilation-arm.md | 645 --
.../install/source-install/compilation-general.md | 277 -
.../docs/install/source-install/compilation-mac.md | 118 -
.../docs/install/source-install/compilation-win.md | 94 -
.../compilation-with-ldb-toolchain.md | 144 -
docs/zh-CN/docs/install/standard-deployment.md | 391 -
docs/zh-CN/docs/lakehouse/cloud-auth/cloud-auth.md | 61 -
docs/zh-CN/docs/lakehouse/compute-node.md | 143 -
docs/zh-CN/docs/lakehouse/external-statistics.md | 49 -
docs/zh-CN/docs/lakehouse/faq.md | 298 -
docs/zh-CN/docs/lakehouse/file.md | 201 -
docs/zh-CN/docs/lakehouse/filecache.md | 96 -
docs/zh-CN/docs/lakehouse/fs-benchmark-tool.md | 231 -
docs/zh-CN/docs/lakehouse/multi-catalog/dlf.md | 111 -
docs/zh-CN/docs/lakehouse/multi-catalog/es.md | 504 --
docs/zh-CN/docs/lakehouse/multi-catalog/hive.md | 557 --
docs/zh-CN/docs/lakehouse/multi-catalog/hudi.md | 111 -
docs/zh-CN/docs/lakehouse/multi-catalog/iceberg.md | 231 -
docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md | 1056 ---
.../docs/lakehouse/multi-catalog/max-compute.md | 90 -
.../docs/lakehouse/multi-catalog/multi-catalog.md | 326 -
docs/zh-CN/docs/lakehouse/multi-catalog/paimon.md | 197 -
docs/zh-CN/docs/lakehouse/sql-dialect.md | 128 -
.../async-materialized-view.md | 214 -
.../query-async-materialized-view.md | 417 -
.../hight-concurrent-point-query.md | 159 -
.../zh-CN/docs/query-acceleration/hint/joinHint.md | 610 --
.../join-optimization/bucket-shuffle-join.md | 106 -
.../join-optimization/colocation-join.md | 432 --
.../join-optimization/doris-join-optimization.md | 226 -
.../join-optimization/runtime-filter.md | 289 -
.../docs/query-acceleration/materialized-view.md | 541 --
docs/zh-CN/docs/query-acceleration/nereids.md | 86 -
.../pipeline-execution-engine.md | 79 -
.../pipeline-x-execution-engine.md | 75 -
docs/zh-CN/docs/query-acceleration/statistics.md | 343 -
docs/zh-CN/docs/releasenotes/release-1.1.0.md | 393 -
docs/zh-CN/docs/releasenotes/release-1.1.1.md | 81 -
docs/zh-CN/docs/releasenotes/release-1.1.2.md | 91 -
docs/zh-CN/docs/releasenotes/release-1.1.3.md | 91 -
docs/zh-CN/docs/releasenotes/release-1.1.4.md | 71 -
docs/zh-CN/docs/releasenotes/release-1.1.5.md | 68 -
docs/zh-CN/docs/releasenotes/release-1.2.0.md | 621 --
docs/zh-CN/docs/releasenotes/release-1.2.1.md | 193 -
docs/zh-CN/docs/releasenotes/release-1.2.2.md | 260 -
docs/zh-CN/docs/releasenotes/release-1.2.3.md | 111 -
docs/zh-CN/docs/releasenotes/release-1.2.4.md | 177 -
docs/zh-CN/docs/releasenotes/release-1.2.5.md | 200 -
docs/zh-CN/docs/releasenotes/release-1.2.6.md | 135 -
docs/zh-CN/docs/releasenotes/release-1.2.7.md | 47 -
docs/zh-CN/docs/releasenotes/release-1.2.8.md | 48 -
docs/zh-CN/docs/releasenotes/release-2.0-Alpha.md | 129 -
docs/zh-CN/docs/releasenotes/release-2.0-Beta.md | 303 -
docs/zh-CN/docs/releasenotes/release-2.0.0.md | 221 -
docs/zh-CN/docs/releasenotes/release-2.0.1.md | 158 -
docs/zh-CN/docs/releasenotes/release-2.0.2.md | 210 -
docs/zh-CN/docs/releasenotes/release-2.0.3.md | 295 -
docs/zh-CN/docs/releasenotes/release-2.0.4.md | 60 -
docs/zh-CN/docs/releasenotes/release-2.0.5.md | 68 -
docs/zh-CN/docs/releasenotes/release-2.0.6.md | 54 -
docs/zh-CN/docs/releasenotes/release-2.1.0.md | 892 ---
.../sql-functions/string-functions/repeat.md | 7 +
.../Create/CREATE-EXTERNAL-TABLE.md | 272 +
.../sql-reference/Data-Types/AGG_STATE.md | 4 +-
.../java/org/apache/doris/avro/AvroJNIScanner.java | 3 +-
.../java/org/apache/doris/hudi/HudiJniScanner.java | 28 +-
.../org/apache/doris/hudi/BaseSplitReader.scala | 8 +-
.../apache/doris/hudi/HoodieRecordIterator.scala | 67 +-
.../doris/hudi/MORIncrementalSplitReader.scala | 86 +
.../apache/doris/hudi/MORSnapshotSplitReader.scala | 2 +-
.../org/apache/doris/common/jni/JniScanner.java | 12 +-
.../apache/doris/common/jni/MockJniScanner.java | 11 +-
.../org/apache/doris/jdbc/BaseJdbcExecutor.java | 2 +
.../org/apache/doris/jdbc/DefaultJdbcExecutor.java | 2 +
.../doris/maxcompute/MaxComputeJniScanner.java | 16 +-
.../org/apache/doris/paimon/PaimonJniScanner.java | 3 +-
.../trinoconnector/TrinoConnectorJniScanner.java | 29 +-
.../org/apache/doris/catalog/AggStateType.java | 4 +-
.../main/java/org/apache/doris/common/Config.java | 31 +-
fe/fe-core/pom.xml | 4 +
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 1 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 25 +-
fe/fe-core/src/main/cup/sql_parser.cup | 49 +-
.../apache/doris/alter/SchemaChangeHandler.java | 8 +
.../org/apache/doris/analysis/BinaryPredicate.java | 11 +-
.../java/org/apache/doris/analysis/ColumnDef.java | 25 +-
.../doris/analysis/CreateMaterializedViewStmt.java | 2 +-
.../org/apache/doris/analysis/DateLiteral.java | 4 +
.../java/org/apache/doris/analysis/LoadStmt.java | 2 +
.../apache/doris/analysis/NativeInsertStmt.java | 5 +-
.../org/apache/doris/analysis/SchemaTableType.java | 3 +-
.../apache/doris/analysis/SetUserPropertyVar.java | 11 +
.../org/apache/doris/analysis/ShowTableStmt.java | 2 +
.../org/apache/doris/analysis/StmtRewriter.java | 15 +-
.../java/org/apache/doris/analysis/TableRef.java | 14 +
.../org/apache/doris/analysis/TableScanParams.java | 46 +
.../org/apache/doris/catalog/AggregateType.java | 2 +-
.../doris/catalog/BuiltinTableValuedFunctions.java | 4 +-
.../apache/doris/catalog/ColocateTableIndex.java | 5 +
.../main/java/org/apache/doris/catalog/Column.java | 35 +-
.../main/java/org/apache/doris/catalog/Env.java | 8 +-
.../java/org/apache/doris/catalog/EnvFactory.java | 23 +
.../java/org/apache/doris/catalog/OlapTable.java | 34 +
.../java/org/apache/doris/catalog/SchemaTable.java | 19 +-
.../main/java/org/apache/doris/catalog/Table.java | 1 +
.../org/apache/doris/catalog/TableProperty.java | 15 +
.../authorizer/ranger/RangerAccessController.java | 78 +
.../ranger/doris/RangerDorisAccessController.java | 58 +-
.../ranger/hive/RangerHiveAccessController.java | 75 +-
.../org/apache/doris/cloud/catalog/CloudEnv.java | 2 +-
.../doris/cloud/catalog/CloudEnvFactory.java | 30 +
.../cloud/datasource/CloudInternalCatalog.java | 4 +
.../doris/cloud/load/CloudBrokerLoadJob.java | 181 +
.../org/apache/doris/cloud/load/CloudCopyJob.java | 106 +
.../apache/doris/cloud/load/CloudLoadManager.java | 49 +
.../doris/cloud/load/CloudRoutineLoadManager.java | 66 +
.../cloud/planner/CloudGroupCommitPlanner.java | 81 +
.../apache/doris/cloud/qe/CloudCoordinator.java | 4 +-
.../doris/cloud/system/CloudSystemInfoService.java | 43 +-
.../transaction/CloudGlobalTransactionMgr.java | 2 +-
.../java/org/apache/doris/common/ErrorCode.java | 5 +-
.../org/apache/doris/common/InternalErrorCode.java | 1 +
.../doris/common/profile/AggregatedProfile.java | 41 -
.../org/apache/doris/common/profile/Profile.java | 40 +-
.../apache/doris/common/util/PropertyAnalyzer.java | 37 +-
.../org/apache/doris/datasource/CatalogIf.java | 3 +
.../apache/doris/datasource/ExternalCatalog.java | 16 +
.../apache/doris/datasource/ExternalScanNode.java | 3 +-
.../doris/datasource/FederationBackendPolicy.java | 6 +-
.../apache/doris/datasource/InternalCatalog.java | 13 +-
.../doris/datasource/hive/HMSCachedClient.java | 18 +
.../apache/doris/datasource/hive/HMSCommitter.java | 668 ++
.../doris/datasource/hive/HMSExternalTable.java | 100 +-
.../datasource/hive/HiveColumnStatistics.java | 30 +
.../datasource/hive/HiveCommonStatistics.java | 44 +
.../doris/datasource/hive/HiveMetadataOps.java | 46 +-
.../doris/datasource/hive/HivePartition.java | 16 +
.../datasource/hive/HivePartitionStatistics.java | 117 +
.../hive/HivePartitionWithStatistics.java | 42 +
.../doris/datasource/hive/HiveTableMetadata.java | 30 +-
.../org/apache/doris/datasource/hive/HiveUtil.java | 64 +-
.../hive/PostgreSQLJdbcHMSCachedClient.java | 59 +-
.../datasource/hive/ThriftHMSCachedClient.java | 185 +-
.../hudi/source/COWIncrementalRelation.java | 254 +
.../doris/datasource/hudi/source/HudiScanNode.java | 175 +-
.../hudi/source/IncrementalRelation.java | 44 +
.../hudi/source/MORIncrementalRelation.java | 217 +
.../doris/datasource/iceberg/IcebergUtils.java | 2 +-
.../doris/datasource/jdbc/client/JdbcClient.java | 22 +-
.../datasource/jdbc/client/JdbcMySQLClient.java | 25 +-
.../datasource/jdbc/client/JdbcOracleClient.java | 16 +
.../TrinoConnectorExternalTable.java | 2 +-
.../source/TrinoConnectorScanNode.java | 15 +-
.../main/java/org/apache/doris/fs/FileSystem.java | 21 +
.../apache/doris/fs/remote/dfs/DFSFileSystem.java | 89 +-
.../org/apache/doris/httpv2/rest/LoadAction.java | 139 +-
.../doris/httpv2/rest/manager/ClusterAction.java | 37 +
.../java/org/apache/doris/load/EtlJobType.java | 1 +
.../apache/doris/load/loadv2/BrokerLoadJob.java | 28 +-
.../org/apache/doris/load/loadv2/JobState.java | 3 +-
.../java/org/apache/doris/load/loadv2/LoadJob.java | 8 +-
.../apache/doris/load/loadv2/LoadLoadingTask.java | 3 +-
.../org/apache/doris/load/loadv2/LoadManager.java | 12 +-
.../org/apache/doris/load/loadv2/LoadTask.java | 23 +
.../apache/doris/load/loadv2/MysqlLoadManager.java | 11 +
.../doris/load/routineload/KafkaTaskInfo.java | 2 +
.../doris/load/routineload/RoutineLoadJob.java | 70 +-
.../doris/load/routineload/RoutineLoadManager.java | 5 +-
.../doris/load/routineload/ScheduleRule.java | 5 +-
.../java/org/apache/doris/metric/MetricRepo.java | 16 +-
.../org/apache/doris/mysql/AcceptListener.java | 14 +
.../java/org/apache/doris/mysql/BytesChannel.java | 29 +
.../java/org/apache/doris/mysql/MysqlChannel.java | 32 +-
.../java/org/apache/doris/mysql/MysqlProto.java | 1 -
.../apache/doris/mysql/ProxyProtocolHandler.java | 212 +
.../mysql/privilege/AccessControllerManager.java | 33 +-
.../org/apache/doris/mysql/privilege/Auth.java | 4 +-
.../mysql/privilege/CatalogAccessController.java | 7 +
.../doris/mysql/privilege/DataMaskPolicy.java | 24 +
.../mysql/privilege/InternalAccessController.java | 17 +-
.../mysql/privilege/RangerDataMaskPolicy.java | 103 +
.../mysql/privilege/RangerRowFilterPolicy.java | 95 +
.../doris/mysql/privilege/RowFilterPolicy.java | 32 +
.../nereids/analyzer/UnboundHiveTableSink.java | 149 +
.../doris/nereids/analyzer/UnboundRelation.java | 34 +-
.../doris/nereids/analyzer/UnboundTableSink.java | 49 +-
.../nereids/analyzer/UnboundTableSinkCreator.java | 74 +
.../glue/translator/PhysicalPlanTranslator.java | 63 +-
.../org/apache/doris/nereids/hint/LeadingHint.java | 5 +
.../doris/nereids/jobs/executor/Analyzer.java | 2 +
.../doris/nereids/jobs/executor/Rewriter.java | 4 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 57 +-
.../processor/pre/PullUpSubqueryAliasToCTE.java | 8 +
.../properties/ChildrenPropertiesRegulator.java | 22 +-
.../DistributionSpecTableSinkHashPartitioned.java | 50 +
...DistributionSpecTableSinkRandomPartitioned.java | 36 +
.../nereids/properties/PhysicalProperties.java | 3 +
.../nereids/properties/RequestPropertyDeriver.java | 11 +
.../org/apache/doris/nereids/rules/RuleType.java | 2 +-
.../doris/nereids/rules/analysis/BindRelation.java | 9 +-
.../doris/nereids/rules/analysis/BindSink.java | 569 +-
.../doris/nereids/rules/analysis/CheckPolicy.java | 25 +-
.../rules/analysis/CollectJoinConstraint.java | 12 +-
.../rules/analysis/EliminateLogicalSelectHint.java | 12 +-
.../nereids/rules/analysis/ExpressionAnalyzer.java | 20 +
.../nereids/rules/analysis/NormalizeAggregate.java | 44 +-
.../nereids/rules/analysis/NormalizeRepeat.java | 11 +
.../nereids/rules/analysis/SubExprAnalyzer.java | 17 +
.../join/PushDownProjectThroughSemiJoin.java | 19 +
.../rules/expression/ExpressionNormalization.java | 2 +
.../expression/rules/ConvertAggStateCast.java | 81 +
.../rules/expression/rules/FunctionBinder.java | 20 +
...ogicalHiveTableSinkToPhysicalHiveTableSink.java | 4 +-
.../doris/nereids/rules/rewrite/ColumnPruning.java | 29 +-
.../rules/rewrite/EliminateAssertNumRows.java | 13 +-
.../nereids/rules/rewrite/EliminateGroupByKey.java | 5 -
.../nereids/rules/rewrite/ScalarApplyToJoin.java | 7 +-
.../mv/SelectMaterializedIndexWithAggregate.java | 2 +-
.../trees/expressions/functions/scalar/Repeat.java | 4 +-
.../functions/table/WorkloadGroups.java | 56 -
.../visitor/TableValuedFunctionVisitor.java | 5 -
.../apache/doris/nereids/trees/plans/PlanType.java | 2 +
.../doris/nereids/trees/plans/algebra/Repeat.java | 18 +-
.../trees/plans/commands/CreateTableCommand.java | 6 +-
.../plans/commands/DeleteFromUsingCommand.java | 4 +-
.../nereids/trees/plans/commands/LoadCommand.java | 4 +-
.../trees/plans/commands/UpdateCommand.java | 6 +-
.../plans/commands/UpdateMvByPartitionCommand.java | 11 +-
.../plans/commands/info/ColumnDefinition.java | 64 +-
.../trees/plans/commands/info/DefaultValue.java | 2 +
.../commands/insert/AbstractInsertExecutor.java | 2 +-
.../plans/commands/insert/GroupCommitInserter.java | 4 +-
.../commands/insert/HiveInsertCommandContext.java | 33 +
.../plans/commands/insert/HiveInsertExecutor.java | 45 +-
.../commands/insert/InsertIntoTableCommand.java | 6 +-
.../insert/InsertOverwriteTableCommand.java | 56 +-
.../trees/plans/commands/insert/InsertUtils.java | 76 +-
.../trees/plans/logical/LogicalAssertNumRows.java | 5 +-
.../trees/plans/logical/LogicalCheckPolicy.java | 35 +-
.../trees/plans/logical/LogicalHiveTableSink.java | 41 +-
.../nereids/trees/plans/logical/LogicalRepeat.java | 4 +
.../trees/plans/logical/UnboundLogicalSink.java | 62 +
.../trees/plans/physical/AbstractPhysicalPlan.java | 10 -
.../plans/physical/PhysicalAssertNumRows.java | 5 +-
.../plans/physical/PhysicalHiveTableSink.java | 78 +-
.../nereids/trees/plans/physical/PhysicalSink.java | 10 +-
.../trees/plans/physical/PhysicalTableSink.java | 2 +
.../nereids/trees/plans/visitor/SinkVisitor.java | 5 +
.../apache/doris/nereids/types/AggStateType.java | 19 +-
.../apache/doris/nereids/util/RelationUtil.java | 2 +-
.../doris/nereids/util/TypeCoercionUtils.java | 8 +
.../apache/doris/planner/AssertNumRowsNode.java | 22 +-
.../org/apache/doris/planner/DataPartition.java | 2 +
.../apache/doris/planner/GroupCommitPlanner.java | 59 +-
.../org/apache/doris/planner/HiveTableSink.java | 172 +-
.../org/apache/doris/planner/OlapScanNode.java | 13 +-
.../org/apache/doris/planner/OlapTableSink.java | 12 +-
.../org/apache/doris/plugin/audit/AuditEvent.java | 7 +
.../org/apache/doris/policy/DropPolicyLog.java | 42 +-
.../main/java/org/apache/doris/policy/Policy.java | 11 +-
.../java/org/apache/doris/policy/PolicyMgr.java | 126 +-
.../java/org/apache/doris/policy/RowPolicy.java | 69 +-
.../java/org/apache/doris/qe/AuditLogHelper.java | 4 +
.../java/org/apache/doris/qe/ConnectContext.java | 24 +-
.../main/java/org/apache/doris/qe/Coordinator.java | 9 +-
.../java/org/apache/doris/qe/MasterOpExecutor.java | 6 +
.../java/org/apache/doris/qe/QeProcessorImpl.java | 29 +-
.../org/apache/doris/qe/QueryStatisticsItem.java | 6 +-
.../java/org/apache/doris/qe/SessionVariable.java | 14 +
.../java/org/apache/doris/qe/ShowExecutor.java | 12 +-
.../java/org/apache/doris/qe/StmtExecutor.java | 185 +-
.../doris/resource/workloadgroup/QueryQueue.java | 6 +-
.../doris/resource/workloadgroup/QueueToken.java | 33 +-
.../apache/doris/rewrite/FoldConstantsRule.java | 3 +-
.../apache/doris/service/FrontendServiceImpl.java | 102 +-
.../main/java/org/apache/doris/system/Backend.java | 22 +
.../org/apache/doris/system/BackendHbResponse.java | 15 +-
.../org/apache/doris/system/BeSelectionPolicy.java | 4 +-
.../java/org/apache/doris/system/HeartbeatMgr.java | 7 +-
.../org/apache/doris/system/SystemInfoService.java | 27 +-
.../doris/tablefunction/MetadataGenerator.java | 178 +-
.../tablefunction/MetadataTableValuedFunction.java | 2 -
.../doris/tablefunction/TableValuedFunctionIf.java | 2 -
.../WorkloadGroupsTableValuedFunction.java | 99 -
.../org/apache/doris/task/CreateReplicaTask.java | 10 +
fe/fe-core/src/main/jflex/sql_scanner.flex | 2 +
.../analysis/CreateTableAsSelectStmtTest.java | 21 +
.../apache/doris/analysis/ShowTableStmtTest.java | 4 +-
.../apache/doris/datasource/ColumnPrivTest.java | 15 +
.../doris/datasource/RefreshCatalogTest.java | 4 +-
.../doris/datasource/hive/HmsCommitTest.java | 245 +
.../load/routineload/RoutineLoadManagerTest.java | 6 +-
.../load/routineload/RoutineLoadSchedulerTest.java | 4 +-
.../nereids/privileges/TestCheckPrivileges.java | 15 +
.../rules/analysis/FillUpMissingSlotsTest.java | 4 +-
.../join/PushDownProjectThroughSemiJoinTest.java | 32 +
.../rules/rewrite/SimplifyAggGroupByTest.java | 15 +
.../apache/doris/planner/StatisticDeriveTest.java | 1 -
.../apache/doris/qe/ProxyProtocolHandlerTest.java | 135 +
.../apache/doris/system/SystemInfoServiceTest.java | 2 +-
.../doris/utframe/DemoMultiBackendsTest.java | 3 +-
fe/pom.xml | 18 +-
.../org/apache/doris/load/loadv2/dpp/SparkDpp.java | 11 +-
gensrc/proto/cloud.proto | 3 +
gensrc/proto/olap_file.proto | 7 +
gensrc/script/doris_builtins_functions.py | 4 +-
gensrc/thrift/AgentService.thrift | 10 +-
gensrc/thrift/BackendService.thrift | 2 +
gensrc/thrift/DataSinks.thrift | 34 +-
gensrc/thrift/Descriptors.thrift | 3 +-
gensrc/thrift/Exprs.thrift | 5 +
gensrc/thrift/FrontendService.thrift | 15 +-
gensrc/thrift/PaloInternalService.thrift | 1 +
gensrc/thrift/Partitions.thrift | 8 +-
gensrc/thrift/PlanNodes.thrift | 5 +-
gensrc/thrift/Types.thrift | 2 -
regression-test/conf/regression-conf.groovy | 5 +
.../spark_connector/spark_connector_arrow.out | 14 +-
.../data/correctness_p0/test_current_date.out | 13 +
.../correctness_p0/test_grouping_sets_empty.out | 39 +
.../one_level_nestedtypes_with_s3data.out | 324 +-
.../three_level_nestedtypes_with_s3data.out | 8016 ++++++++++----------
.../two_level_nestedtypes_with_s3data.out | 1992 ++---
.../test_nested_types_insert_into_with_s3.out | 16 +-
.../test_nestedtypes_csv_insert_into_with_s3.out | 6 +-
.../test_nestedtypes_json_insert_into_with_s3.out | 10 +-
.../outfile/csv/test_outfile_csv_array_type.out | 14 +-
.../outfile/csv/test_outfile_csv_map_type.out | 12 +-
.../parquet/test_outfile_parquet_array_type.out | 28 +-
.../parquet/test_outfile_parquet_map_type.out | 24 +-
.../data/export_p0/test_outfile_orc_array_type.out | 28 +-
.../data/export_p0/test_outfile_orc_map_type.out | 24 +-
.../hive/test_hive_basic_type.out | 6 +-
.../data/external_table_p0/hive/test_hive_orc.out | 132 +-
.../jdbc/test_mariadb_jdbc_catalog.out | 1 +
.../jdbc/test_mysql_jdbc_catalog.out | 6 +-
.../jdbc/test_mysql_jdbc_catalog_nereids.out | 1 +
.../jdbc/test_mysql_jdbc_driver5_catalog.out | 5 +-
.../jdbc/test_sqlserver_jdbc_catalog.out | 1 -
.../paimon/test_paimon_catalog.out | 6 +-
.../tvf/test_local_tvf_with_complex_type.out | 102 +-
...test_local_tvf_with_complex_type_element_at.out | 134 +-
...ocal_tvf_with_complex_type_insertinto_doris.out | 306 +-
.../data/external_table_p2/hive/test_hive_hudi.out | 12 +
.../data/function_p0/test_agg_foreach.out | 2 +-
.../index_change/test_index_change_1.out | 25 +
.../index_change/test_index_change_2.out | 42 +
.../index_change/test_index_change_3.out | 42 +
.../index_change/test_index_change_4.out | 42 +
.../index_change/test_index_change_5.out | 42 +
.../index_change/test_index_change_6.out | 42 +
.../storage_format}/documents-1000.json | 0
.../storage_format/test_storage_format_v1.out | 16 +
.../data/jsonb_p0/test_jsonb_load_and_function.out | 3 +
.../http_stream/test_http_stream_properties.out | 204 +-
.../load_p0/routine_load/test_routine_load.out | 577 +-
.../stream_load/test_stream_load_properties.out | 346 +-
.../data/nereids_clickbench_shape_p0/query1.out | 9 +
.../data/nereids_clickbench_shape_p0/query10.out | 12 +
.../data/nereids_clickbench_shape_p0/query11.out | 13 +
.../data/nereids_clickbench_shape_p0/query12.out | 13 +
.../data/nereids_clickbench_shape_p0/query13.out | 13 +
.../data/nereids_clickbench_shape_p0/query14.out | 13 +
.../data/nereids_clickbench_shape_p0/query15.out | 13 +
.../data/nereids_clickbench_shape_p0/query16.out | 10 +
.../data/nereids_clickbench_shape_p0/query17.out | 10 +
.../data/nereids_clickbench_shape_p0/query18.out | 10 +
.../data/nereids_clickbench_shape_p0/query19.out | 10 +
.../data/nereids_clickbench_shape_p0/query2.out | 10 +
.../data/nereids_clickbench_shape_p0/query20.out | 8 +
.../data/nereids_clickbench_shape_p0/query21.out | 10 +
.../data/nereids_clickbench_shape_p0/query22.out | 13 +
.../data/nereids_clickbench_shape_p0/query23.out | 13 +
.../data/nereids_clickbench_shape_p0/query24.out | 9 +
.../data/nereids_clickbench_shape_p0/query25.out | 11 +
.../data/nereids_clickbench_shape_p0/query26.out | 10 +
.../data/nereids_clickbench_shape_p0/query27.out | 11 +
.../data/nereids_clickbench_shape_p0/query28.out | 14 +
.../data/nereids_clickbench_shape_p0/query29.out | 14 +
.../data/nereids_clickbench_shape_p0/query3.out | 9 +
.../data/nereids_clickbench_shape_p0/query30.out | 9 +
.../data/nereids_clickbench_shape_p0/query31.out | 13 +
.../data/nereids_clickbench_shape_p0/query32.out | 13 +
.../data/nereids_clickbench_shape_p0/query33.out | 12 +
.../data/nereids_clickbench_shape_p0/query34.out | 12 +
.../data/nereids_clickbench_shape_p0/query35.out | 13 +
.../data/nereids_clickbench_shape_p0/query36.out | 13 +
.../data/nereids_clickbench_shape_p0/query37.out | 13 +
.../data/nereids_clickbench_shape_p0/query38.out | 13 +
.../data/nereids_clickbench_shape_p0/query39.out | 13 +
.../data/nereids_clickbench_shape_p0/query4.out | 9 +
.../data/nereids_clickbench_shape_p0/query40.out | 14 +
.../data/nereids_clickbench_shape_p0/query41.out | 13 +
.../data/nereids_clickbench_shape_p0/query42.out | 13 +
.../data/nereids_clickbench_shape_p0/query43.out | 13 +
.../data/nereids_clickbench_shape_p0/query5.out | 11 +
.../data/nereids_clickbench_shape_p0/query6.out | 12 +
.../data/nereids_clickbench_shape_p0/query7.out | 9 +
.../data/nereids_clickbench_shape_p0/query8.out | 13 +
.../data/nereids_clickbench_shape_p0/query9.out | 12 +
.../nereids_p0/explain/test_pushdown_explain.out | 74 +
.../data/nereids_p0/hint/fix_leading.out | 42 +
.../data/nereids_p0/hint/multi_leading.out | 2 +-
.../data/nereids_p0/subquery/test_subquery.out | 7 +
.../eliminate_gby_key/eliminate_gby_key.out | 164 +
.../grouping_with_same_grouping_set.out | 9 +
.../remove_duplicate_expr_in_grouping_set.out | 19 +
.../nereids_tpcds_shape_sf1000_p0/shape/query9.out | 73 +-
.../noStatsRfPrune/query9.out | 73 +-
.../no_stats_shape/query9.out | 73 +-
.../rf_prune/query44.out | 71 +-
.../rf_prune/query9.out | 73 +-
.../nereids_tpcds_shape_sf100_p0/shape/query44.out | 71 +-
.../nereids_tpcds_shape_sf100_p0/shape/query9.out | 73 +-
.../test_unique_model_schema_value_change.out | 10 +
.../show_p0/test_show_create_table_and_views.out | 8 +-
regression-test/data/variant_p0/view.out | 63 +
.../org/apache/doris/regression/Config.groovy | 11 +
.../apache/doris/regression/ConfigOptions.groovy | 9 +
.../apache/doris/regression/RegressionTest.groovy | 4 +
.../org/apache/doris/regression/suite/Suite.groovy | 2 +-
.../doris/regression/util/TeamcityUtils.groovy | 11 +-
.../cloud_p0/conf/regression-conf-custom.groovy | 15 +-
regression-test/pipeline/p0/conf/be.conf | 54 +-
regression-test/pipeline/p0/conf/fe.conf | 60 +-
.../pipeline/p0/conf/regression-conf.groovy | 6 +-
regression-test/pipeline/p1/conf/be.conf | 66 +-
regression-test/pipeline/p1/conf/fe.conf | 86 +-
regression-test/plugins/plugin_cluster.groovy | 180 +
.../test_vertical_compaction_agg_state.groovy | 2 +-
.../suites/correctness/test_select_except.groovy | 2 +-
.../suites/correctness_p0/test_current_date.groovy | 75 +
.../correctness_p0/test_grouping_sets_empty.groovy | 79 +
.../agg_state/avg/test_agg_state_avg.groovy | 2 +-
.../test_agg_state_group_concat.groovy | 2 +-
.../agg_state/max/test_agg_state_max.groovy | 4 +-
.../nereids/test_agg_state_nereids.groovy | 5 +-
.../test_agg_state_quantile_union.groovy | 2 +-
.../datatype_p0/agg_state/test_agg_state.groovy | 2 +-
.../suites/delete_p0/test_delete.groovy | 37 +
.../external_table_p2/hive/test_hive_hudi.groovy | 6 +
...index_compound_directory_fault_injection.groovy | 28 +-
.../index_change/test_index_change_1.groovy | 67 +
.../index_change/test_index_change_2.groovy | 70 +
.../index_change/test_index_change_3.groovy | 73 +
.../index_change/test_index_change_4.groovy | 72 +
.../index_change/test_index_change_5.groovy | 70 +
.../index_change/test_index_change_6.groovy | 78 +
.../storage_format/test_storage_format_v1.groovy | 116 +
.../suites/inverted_index_p0/test_show_data.groovy | 177 +
.../jsonb_p0/test_jsonb_load_and_function.groovy | 1 +
.../suites/mv_p0/dis_26495/dis_26495.groovy | 2 +-
.../suites/nereids_clickbench_shape_p0/load.groovy | 146 +
.../nereids_clickbench_shape_p0/query1.groovy | 29 +
.../nereids_clickbench_shape_p0/query10.groovy | 30 +
.../nereids_clickbench_shape_p0/query11.groovy | 31 +
.../nereids_clickbench_shape_p0/query12.groovy | 31 +
.../nereids_clickbench_shape_p0/query13.groovy | 31 +
.../nereids_clickbench_shape_p0/query14.groovy | 31 +
.../nereids_clickbench_shape_p0/query15.groovy | 31 +
.../nereids_clickbench_shape_p0/query16.groovy | 31 +
.../nereids_clickbench_shape_p0/query17.groovy | 31 +
.../nereids_clickbench_shape_p0/query18.groovy | 31 +
.../nereids_clickbench_shape_p0/query19.groovy | 31 +
.../nereids_clickbench_shape_p0/query2.groovy | 31 +
.../nereids_clickbench_shape_p0/query20.groovy | 31 +
.../nereids_clickbench_shape_p0/query21.groovy | 31 +
.../nereids_clickbench_shape_p0/query22.groovy | 31 +
.../nereids_clickbench_shape_p0/query23.groovy | 31 +
.../nereids_clickbench_shape_p0/query24.groovy | 31 +
.../nereids_clickbench_shape_p0/query25.groovy | 31 +
.../nereids_clickbench_shape_p0/query26.groovy | 31 +
.../nereids_clickbench_shape_p0/query27.groovy | 31 +
.../nereids_clickbench_shape_p0/query28.groovy | 31 +
.../nereids_clickbench_shape_p0/query29.groovy | 31 +
.../nereids_clickbench_shape_p0/query3.groovy | 29 +
.../nereids_clickbench_shape_p0/query30.groovy | 31 +
.../nereids_clickbench_shape_p0/query31.groovy | 31 +
.../nereids_clickbench_shape_p0/query32.groovy | 31 +
.../nereids_clickbench_shape_p0/query33.groovy | 31 +
.../nereids_clickbench_shape_p0/query34.groovy | 31 +
.../nereids_clickbench_shape_p0/query35.groovy | 31 +
.../nereids_clickbench_shape_p0/query36.groovy | 31 +
.../nereids_clickbench_shape_p0/query37.groovy | 31 +
.../nereids_clickbench_shape_p0/query38.groovy | 31 +
.../nereids_clickbench_shape_p0/query39.groovy | 31 +
.../nereids_clickbench_shape_p0/query4.groovy | 29 +
.../nereids_clickbench_shape_p0/query40.groovy | 31 +
.../nereids_clickbench_shape_p0/query41.groovy | 31 +
.../nereids_clickbench_shape_p0/query42.groovy | 31 +
.../nereids_clickbench_shape_p0/query43.groovy | 30 +
.../nereids_clickbench_shape_p0/query5.groovy | 29 +
.../nereids_clickbench_shape_p0/query6.groovy | 29 +
.../nereids_clickbench_shape_p0/query7.groovy | 30 +
.../nereids_clickbench_shape_p0/query8.groovy | 30 +
.../nereids_clickbench_shape_p0/query9.groovy | 31 +
.../explain/test_pushdown_explain.groovy | 86 +
.../suites/nereids_p0/hint/fix_leading.groovy | 17 +
.../nereids_p0/subquery/test_subquery.groovy | 39 +-
.../eliminate_gby_key/eliminate_gby_key.groovy | 251 +
.../grouping_with_same_grouping_set.groovy | 31 +
.../remove_duplicate_expr_in_grouping_set.groovy | 37 +
.../grouping_sets/test_grouping_sets.groovy | 2 +-
.../schema_table/test_active_queries.groovy | 6 +-
.../test_array_functions_with_where.groovy | 2 +-
.../query_p0/sql_functions/test_in_expr.groovy | 2 +-
.../test_alter_colocate_table.groovy | 3 +
.../test_unique_model_schema_value_change.groovy | 410 +-
.../suites/statistics/test_agg_complex_type.groovy | 2 +-
.../storage_medium_p0/test_disk_health.groovy | 4 +
.../test_storage_medium_has_disk.groovy | 4 +
...test_partial_update_insert_schema_change.groovy | 2 +-
regression-test/suites/variant_p0/view.groovy | 83 +
.../workload_manager_p0/test_curd_wlg.groovy | 12 +-
1366 files changed, 36380 insertions(+), 155977 deletions(-)
delete mode 100644 be/src/cloud/cloud_vertical_rowset_writer.cpp
delete mode 100644 be/src/cloud/cloud_vertical_rowset_writer.h
create mode 100644 be/src/exec/schema_scanner/schema_workload_groups_scanner.cpp
create mode 100644 be/src/exec/schema_scanner/schema_workload_groups_scanner.h
create mode 100644 be/src/http/action/clear_file_cache_action.cpp
create mode 100644 be/src/http/action/clear_file_cache_action.h
delete mode 100644 be/src/io/cache/block/block_file_cache.cpp
delete mode 100644 be/src/io/cache/block/block_file_cache.h
delete mode 100644 be/src/io/cache/block/block_file_cache_factory.cpp
delete mode 100644 be/src/io/cache/block/block_file_cache_factory.h
delete mode 100644 be/src/io/cache/block/block_file_cache_fwd.h
delete mode 100644 be/src/io/cache/block/block_file_cache_profile.cpp
delete mode 100644 be/src/io/cache/block/block_file_cache_profile.h
delete mode 100644 be/src/io/cache/block/block_file_cache_settings.h
delete mode 100644 be/src/io/cache/block/block_file_segment.cpp
delete mode 100644 be/src/io/cache/block/block_file_segment.h
delete mode 100644 be/src/io/cache/block/block_lru_file_cache.cpp
delete mode 100644 be/src/io/cache/block/block_lru_file_cache.h
delete mode 100644 be/src/io/cache/block/cached_remote_file_reader.cpp
delete mode 100644 be/src/io/cache/block/cached_remote_file_reader.h
create mode 100644 be/src/io/cache/block_file_cache.cpp
create mode 100644 be/src/io/cache/block_file_cache.h
create mode 100644 be/src/io/cache/block_file_cache_factory.cpp
create mode 100644 be/src/io/cache/block_file_cache_factory.h
create mode 100644 be/src/io/cache/block_file_cache_profile.cpp
create mode 100644 be/src/io/cache/block_file_cache_profile.h
create mode 100644 be/src/io/cache/cached_remote_file_reader.cpp
create mode 100644 be/src/io/cache/cached_remote_file_reader.h
create mode 100644 be/src/io/cache/file_block.cpp
create mode 100644 be/src/io/cache/file_block.h
create mode 100644 be/src/io/cache/file_cache_common.cpp
create mode 100644 be/src/io/cache/file_cache_common.h
create mode 100644 be/src/io/cache/file_cache_storage.h
create mode 100644 be/src/io/cache/fs_file_cache_storage.cpp
create mode 100644 be/src/io/cache/fs_file_cache_storage.h
delete mode 100644 be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp
delete mode 100644 be/src/olap/rowset/segment_v2/inverted_index_compound_directory.h
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index_file_reader.h
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index_file_writer.h
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h
delete mode 100644 be/src/olap/rowset/vertical_beta_rowset_writer_helper.cpp
delete mode 100644 be/src/olap/rowset/vertical_beta_rowset_writer_helper.h
create mode 100644 be/src/pipeline/exec/hive_table_sink_operator.cpp
create mode 100644 be/src/pipeline/exec/hive_table_sink_operator.h
delete mode 100644 be/src/runtime/task_group/task_group.cpp
delete mode 100644 be/src/runtime/task_group/task_group.h
delete mode 100644 be/src/runtime/task_group/task_group_manager.cpp
delete mode 100644 be/src/runtime/task_group/task_group_manager.h
create mode 100644 be/src/runtime/workload_group/workload_group.cpp
create mode 100644 be/src/runtime/workload_group/workload_group.h
create mode 100644 be/src/runtime/workload_group/workload_group_manager.cpp
create mode 100644 be/src/runtime/workload_group/workload_group_manager.h
create mode 100644 be/src/util/indexed_priority_queue.hpp
create mode 100644 be/src/util/profile_collector.h
create mode 100644 be/src/vec/exec/skewed_partition_rebalancer.cpp
create mode 100644 be/src/vec/exec/skewed_partition_rebalancer.h
create mode 100644 be/src/vec/sink/scale_writer_partitioning_exchanger.hpp
create mode 100644 be/src/vec/sink/vhive_table_sink.cpp
create mode 100644 be/src/vec/sink/vhive_table_sink.h
create mode 100644 be/src/vec/sink/writer/vhive_partition_writer.cpp
create mode 100644 be/src/vec/sink/writer/vhive_partition_writer.h
create mode 100644 be/src/vec/sink/writer/vhive_table_writer.cpp
create mode 100644 be/src/vec/sink/writer/vhive_table_writer.h
create mode 100644 be/src/vec/sink/writer/vhive_utils.cpp
create mode 100644 be/src/vec/sink/writer/vhive_utils.h
create mode 100644 be/test/io/cache/block_file_cache_test.cpp
delete mode 100644 be/test/io/cache/file_block_cache_test.cpp
create mode 100644 be/test/util/indexed_priority_queue_test.cpp
create mode 100644 be/test/vec/data_types/datetime_round_test.cpp
create mode 100644 be/test/vec/exec/skewed_partition_rebalancer_test.cpp
create mode 100644 be/test/vec/exec/vhive_utils_test.cpp
delete mode 100644 docs/en/community/design/doris_storage_optimization.md
delete mode 100644 docs/en/community/design/grouping_sets_design.md
delete mode 100644 docs/en/community/design/metadata-design.md
delete mode 100644 docs/en/community/design/spark_load.md
delete mode 100644 docs/en/community/developer-guide/be-clion-dev.md
delete mode 100644 docs/en/community/developer-guide/be-vscode-dev.md
delete mode 100644 docs/en/community/developer-guide/benchmark-tool.md
delete mode 100644 docs/en/community/developer-guide/bitmap-hll-file-format.md
delete mode 100644 docs/en/community/developer-guide/cpp-diagnostic-code.md
delete mode 100644 docs/en/community/developer-guide/cpp-format-code.md
delete mode 100644 docs/en/community/developer-guide/debug-tool.md
delete mode 100644 docs/en/community/developer-guide/docker-dev.md
delete mode 100644 docs/en/community/developer-guide/fe-eclipse-dev.md
delete mode 100644 docs/en/community/developer-guide/fe-idea-dev.md
delete mode 100644 docs/en/community/developer-guide/fe-vscode-dev.md
delete mode 100644 docs/en/community/developer-guide/github-checks.md
delete mode 100644 docs/en/community/developer-guide/how-to-share-blogs.md
delete mode 100644 docs/en/community/developer-guide/java-format-code.md
delete mode 100644 docs/en/community/developer-guide/mac-dev/be-clion-dev.md
delete mode 100644 docs/en/community/developer-guide/mac-dev/dev-prepare.md
delete mode 100644 docs/en/community/developer-guide/mac-dev/fe-idea-dev.md
delete mode 100644 docs/en/community/developer-guide/regression-testing.md
delete mode 100644 docs/en/community/doris-future.md
delete mode 100644 docs/en/community/feedback.md
delete mode 100644 docs/en/community/how-to-contribute/commit-format-specification.md
delete mode 100644 docs/en/community/how-to-contribute/contribute-doc.md
delete mode 100644 docs/en/community/how-to-contribute/contributor-guide.md
delete mode 100644 docs/en/community/how-to-contribute/how-to-be-a-committer.md
delete mode 100644 docs/en/community/how-to-contribute/how-to-contribute.md
delete mode 100644 docs/en/community/how-to-contribute/pull-request.md
delete mode 100644 docs/en/community/join-community.md
delete mode 100644 docs/en/community/release-and-verify/release-complete.md
delete mode 100644 docs/en/community/release-and-verify/release-doris-connectors.md
delete mode 100644 docs/en/community/release-and-verify/release-doris-core.md
delete mode 100644 docs/en/community/release-and-verify/release-doris-sdk.md
delete mode 100644 docs/en/community/release-and-verify/release-doris-shade.md
delete mode 100644 docs/en/community/release-and-verify/release-prepare.md
delete mode 100644 docs/en/community/release-and-verify/release-verify.md
delete mode 100644 docs/en/community/security.md
delete mode 100644 docs/en/community/subscribe-mail-list.md
delete mode 100644 docs/en/community/team.md
delete mode 100644 docs/en/docs/admin-manual/certificate.md
delete mode 100644 docs/en/docs/admin-manual/cluster-management/elastic-expansion.md
delete mode 100644 docs/en/docs/admin-manual/cluster-management/fqdn.md
delete mode 100644 docs/en/docs/admin-manual/cluster-management/load-balancing.md
delete mode 100644 docs/en/docs/admin-manual/cluster-management/upgrade.md
delete mode 100644 docs/en/docs/admin-manual/config/be-config.md
delete mode 100644 docs/en/docs/admin-manual/config/config-dir.md
delete mode 100644 docs/en/docs/admin-manual/config/fe-config-template.md
delete mode 100644 docs/en/docs/admin-manual/config/fe-config.md
delete mode 100644 docs/en/docs/admin-manual/config/user-property.md
delete mode 100644 docs/en/docs/admin-manual/data-admin/backup.md
delete mode 100644 docs/en/docs/admin-manual/data-admin/data-recovery.md
delete mode 100644 docs/en/docs/admin-manual/data-admin/delete-recover.md
delete mode 100644 docs/en/docs/admin-manual/data-admin/restore.md
delete mode 100644 docs/en/docs/admin-manual/fe-certificate.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/be/check-rpc-channel.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/be/check-tablet-segment.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/be/checksum.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/be/compaction-run.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/be/compaction-status.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/be/config.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/be/download.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/be/health.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/be/meta.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/be/metrics.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/be/pad-rowset.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/be/reset-rpc-channel.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/be/snapshot.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/be/tablet-distribution.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/be/tablet-info.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/be/tablet-migration.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/be/tablet-reload.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/be/tablet-restore.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/be/version-info.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/backends-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/bootstrap-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/cancel-load-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/check-decommission-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/check-storage-type-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/cluster-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/colocate-meta-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/config-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/connection-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/debug-point-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/extra-basepath-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/fe-version-info-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/get-ddl-stmt-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/get-load-info-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/get-load-state.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/get-log-file-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/get-small-file.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/ha-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/hardware-info-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/health-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/help-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/import-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/log-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/login-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/logout-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/meta-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/meta-info-action-V2.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/meta-info-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/meta-replay-state-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/metrics-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/node-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/profile-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/query-detail-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/query-profile-action-controller.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/query-profile-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/query-schema-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/query-stats-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/row-count-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/session-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/set-config-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/show-data-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/show-meta-info-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/show-proc-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/show-runtime-info-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/show-table-data-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/statement-execution-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/statistic-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/system-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/table-query-plan-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/table-row-count-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/table-schema-action.md
delete mode 100644 docs/en/docs/admin-manual/http-actions/fe/upload-action.md
delete mode 100644 docs/en/docs/admin-manual/maint-monitor/automatic-service-start.md
delete mode 100644 docs/en/docs/admin-manual/maint-monitor/be-olap-error-code.md
delete mode 100644 docs/en/docs/admin-manual/maint-monitor/disk-capacity.md
delete mode 100644 docs/en/docs/admin-manual/maint-monitor/doris-error-code.md
delete mode 100644 docs/en/docs/admin-manual/maint-monitor/memory-management/be-oom-analysis.md
delete mode 100644 docs/en/docs/admin-manual/maint-monitor/memory-management/memory-limit-exceeded-analysis.md
delete mode 100644 docs/en/docs/admin-manual/maint-monitor/memory-management/memory-tracker.md
delete mode 100644 docs/en/docs/admin-manual/maint-monitor/metadata-operation.md
delete mode 100644 docs/en/docs/admin-manual/maint-monitor/monitor-alert.md
delete mode 100644 docs/en/docs/admin-manual/maint-monitor/monitor-metrics/metrics.md
delete mode 100644 docs/en/docs/admin-manual/maint-monitor/multi-tenant.md
delete mode 100644 docs/en/docs/admin-manual/maint-monitor/tablet-local-debug.md
delete mode 100644 docs/en/docs/admin-manual/maint-monitor/tablet-meta-tool.md
delete mode 100644 docs/en/docs/admin-manual/maint-monitor/tablet-repair-and-balance.md
delete mode 100644 docs/en/docs/admin-manual/multi-tenant.md
delete mode 100644 docs/en/docs/admin-manual/privilege-ldap/ldap.md
delete mode 100644 docs/en/docs/admin-manual/privilege-ldap/ranger.md
delete mode 100644 docs/en/docs/admin-manual/privilege-ldap/user-privilege.md
delete mode 100644 docs/en/docs/admin-manual/query-profile.md
delete mode 100644 docs/en/docs/admin-manual/sql-interception.md
delete mode 100644 docs/en/docs/admin-manual/system-table/rowsets.md
delete mode 100644 docs/en/docs/admin-manual/workload-group.md
delete mode 100644 docs/en/docs/advanced/alter-table/replace-table.md
delete mode 100644 docs/en/docs/advanced/alter-table/schema-change.md
delete mode 100644 docs/en/docs/advanced/auto-increment.md
delete mode 100644 docs/en/docs/advanced/autobucket.md
delete mode 100644 docs/en/docs/advanced/best-practice/compaction.md
delete mode 100644 docs/en/docs/advanced/best-practice/debug-log.md
delete mode 100644 docs/en/docs/advanced/best-practice/import-analysis.md
delete mode 100644 docs/en/docs/advanced/best-practice/query-analysis.md
delete mode 100644 docs/en/docs/advanced/broker.md
delete mode 100644 docs/en/docs/advanced/cache/partition-cache-manual.md
delete mode 100644 docs/en/docs/advanced/cache/query-cache.md
delete mode 100644 docs/en/docs/advanced/cache/sql-cache-manual.md
delete mode 100644 docs/en/docs/advanced/cold-hot-separation.md
delete mode 100644 docs/en/docs/advanced/compute-node.md
delete mode 100644 docs/en/docs/advanced/is-being-synced.md
delete mode 100644 docs/en/docs/advanced/lateral-view.md
delete mode 100644 docs/en/docs/advanced/orthogonal-bitmap-manual.md
delete mode 100644 docs/en/docs/advanced/orthogonal-hll-manual.md
delete mode 100644 docs/en/docs/advanced/partition/auto-partition.md
delete mode 100644 docs/en/docs/advanced/partition/dynamic-partition.md
delete mode 100644 docs/en/docs/advanced/partition/table-temp-partition.md
delete mode 100644 docs/en/docs/advanced/resource.md
delete mode 100644 docs/en/docs/advanced/small-file-mgr.md
delete mode 100644 docs/en/docs/advanced/sql-mode.md
delete mode 100644 docs/en/docs/advanced/time-zone.md
delete mode 100644 docs/en/docs/advanced/user-defined-variables.md
delete mode 100644 docs/en/docs/advanced/using-hll.md
delete mode 100644 docs/en/docs/advanced/variables-template.md
delete mode 100644 docs/en/docs/advanced/variables.md
delete mode 100644 docs/en/docs/benchmark/ssb.md
delete mode 100644 docs/en/docs/benchmark/tpch.md
delete mode 100644 docs/en/docs/data-operate/export/export-manual.md
delete mode 100644 docs/en/docs/data-operate/export/export-with-mysql-dump.md
delete mode 100644 docs/en/docs/data-operate/export/outfile.md
delete mode 100644 docs/en/docs/data-operate/import/import-advanced/min-load-replica-num.md
delete mode 100644 docs/en/docs/data-operate/import/import-scenes/external-storage-load.md
delete mode 100644 docs/en/docs/data-operate/import/import-scenes/external-table-load.md
delete mode 100644 docs/en/docs/data-operate/import/import-scenes/jdbc-load.md
delete mode 100644 docs/en/docs/data-operate/import/import-scenes/kafka-load.md
delete mode 100644 docs/en/docs/data-operate/import/import-scenes/load-atomicity.md
delete mode 100644 docs/en/docs/data-operate/import/import-scenes/load-data-convert.md
delete mode 100644 docs/en/docs/data-operate/import/import-scenes/load-strict-mode.md
delete mode 100644 docs/en/docs/data-operate/import/import-scenes/local-file-load.md
delete mode 100644 docs/en/docs/data-operate/import/import-way/binlog-load-manual.md
delete mode 100644 docs/en/docs/data-operate/import/import-way/broker-load-manual.md
delete mode 100644 docs/en/docs/data-operate/import/import-way/group-commit-manual.md
delete mode 100644 docs/en/docs/data-operate/import/import-way/insert-into-manual.md
delete mode 100644 docs/en/docs/data-operate/import/import-way/load-json-format.md
delete mode 100644 docs/en/docs/data-operate/import/import-way/mysql-load-manual.md
delete mode 100644 docs/en/docs/data-operate/import/import-way/routine-load-manual.md
delete mode 100644 docs/en/docs/data-operate/import/import-way/s3-load-manual.md
delete mode 100644 docs/en/docs/data-operate/import/import-way/spark-load-manual.md
delete mode 100644 docs/en/docs/data-operate/import/import-way/stream-load-manual.md
delete mode 100644 docs/en/docs/data-operate/import/load-manual.md
delete mode 100644 docs/en/docs/data-operate/update-delete/batch-delete-manual.md
delete mode 100644 docs/en/docs/data-operate/update-delete/delete-manual.md
delete mode 100644 docs/en/docs/data-operate/update-delete/partial-update.md
delete mode 100644 docs/en/docs/data-operate/update-delete/sequence-column-manual.md
delete mode 100644 docs/en/docs/data-operate/update-delete/update.md
delete mode 100644 docs/en/docs/data-table/advance-usage.md
delete mode 100644 docs/en/docs/data-table/basic-usage.md
delete mode 100644 docs/en/docs/data-table/best-practice.md
delete mode 100644 docs/en/docs/data-table/data-model.md
delete mode 100644 docs/en/docs/data-table/data-partition.md
delete mode 100644 docs/en/docs/data-table/hit-the-rollup.md
delete mode 100644 docs/en/docs/data-table/index/bloomfilter.md
delete mode 100644 docs/en/docs/data-table/index/index-overview.md
delete mode 100644 docs/en/docs/data-table/index/inverted-index.md
delete mode 100644 docs/en/docs/data-table/index/ngram-bloomfilter-index.md
delete mode 100644 docs/en/docs/ecosystem/audit-plugin.md
delete mode 100644 docs/en/docs/ecosystem/beats.md
delete mode 100644 docs/en/docs/ecosystem/bi-integration/apache-superset.md
delete mode 100644 docs/en/docs/ecosystem/cloudcanal.md
delete mode 100644 docs/en/docs/ecosystem/datax.md
delete mode 100644 docs/en/docs/ecosystem/dbt-doris-adapter.md
delete mode 100644 docs/en/docs/ecosystem/doris-kafka-connector.md
delete mode 100644 docs/en/docs/ecosystem/doris-streamloader.md
delete mode 100644 docs/en/docs/ecosystem/flink-doris-connector.md
delete mode 100644 docs/en/docs/ecosystem/hive-bitmap-udf.md
delete mode 100644 docs/en/docs/ecosystem/kyuubi.md
delete mode 100644 docs/en/docs/ecosystem/logstash.md
delete mode 100644 docs/en/docs/ecosystem/plugin-development-manual.md
delete mode 100644 docs/en/docs/ecosystem/seatunnel.md
delete mode 100644 docs/en/docs/ecosystem/spark-doris-connector.md
delete mode 100644 docs/en/docs/ecosystem/udaf/remote-user-defined-aggregation-function.md
delete mode 100644 docs/en/docs/ecosystem/udf/contribute-udf.md
delete mode 100644 docs/en/docs/ecosystem/udf/java-user-defined-function.md
delete mode 100644 docs/en/docs/ecosystem/udf/remote-user-defined-function.md
delete mode 100644 docs/en/docs/faq/data-faq.md
delete mode 100644 docs/en/docs/faq/install-faq.md
delete mode 100644 docs/en/docs/faq/sql-faq.md
delete mode 100644 docs/en/docs/get-starting/quick-start.md
delete mode 100644 docs/en/docs/get-starting/what-is-apache-doris.md
delete mode 100644 docs/en/docs/install/construct-docker/build-docker-image.md
delete mode 100644 docs/en/docs/install/construct-docker/run-docker-cluster.md
delete mode 100644 docs/en/docs/install/k8s-deploy/debug-crash.md
delete mode 100644 docs/en/docs/install/k8s-deploy/doris-cluster-upgrade.md
delete mode 100644 docs/en/docs/install/k8s-deploy/expansion-and-contraction.md
delete mode 100644 docs/en/docs/install/k8s-deploy/helm-chart-deploy.md
delete mode 100644 docs/en/docs/install/k8s-deploy/network.md
delete mode 100644 docs/en/docs/install/k8s-deploy/operator-deploy.md
delete mode 100644 docs/en/docs/install/k8s-deploy/persistent-volume.md
delete mode 100644 docs/en/docs/install/k8s-deploy/root-user-use.md
delete mode 100644 docs/en/docs/install/source-install/compilation-arm.md
delete mode 100644 docs/en/docs/install/source-install/compilation-general.md
delete mode 100644 docs/en/docs/install/source-install/compilation-mac.md
delete mode 100644 docs/en/docs/install/source-install/compilation-win.md
delete mode 100644 docs/en/docs/install/source-install/compilation-with-ldb-toolchain.md
delete mode 100644 docs/en/docs/install/standard-deployment.md
delete mode 100644 docs/en/docs/lakehouse/cloud-auth/cloud-auth.md
delete mode 100644 docs/en/docs/lakehouse/compute-node.md
delete mode 100644 docs/en/docs/lakehouse/external-statistics.md
delete mode 100644 docs/en/docs/lakehouse/faq.md
delete mode 100644 docs/en/docs/lakehouse/file.md
delete mode 100644 docs/en/docs/lakehouse/filecache.md
delete mode 100644 docs/en/docs/lakehouse/fs-benchmark-tool.md
delete mode 100644 docs/en/docs/lakehouse/multi-catalog/dlf.md
delete mode 100644 docs/en/docs/lakehouse/multi-catalog/es.md
delete mode 100644 docs/en/docs/lakehouse/multi-catalog/hive.md
delete mode 100644 docs/en/docs/lakehouse/multi-catalog/hudi.md
delete mode 100644 docs/en/docs/lakehouse/multi-catalog/iceberg.md
delete mode 100644 docs/en/docs/lakehouse/multi-catalog/jdbc.md
delete mode 100644 docs/en/docs/lakehouse/multi-catalog/max-compute.md
delete mode 100644 docs/en/docs/lakehouse/multi-catalog/multi-catalog.md
delete mode 100644 docs/en/docs/lakehouse/multi-catalog/paimon.md
delete mode 100644 docs/en/docs/lakehouse/sql-dialect.md
delete mode 100644 docs/en/docs/query-acceleration/async-materialized-view/async-materialized-view.md
delete mode 100644 docs/en/docs/query-acceleration/async-materialized-view/query-async-materialized-view.md
delete mode 100644 docs/en/docs/query-acceleration/hight-concurrent-point-query.md
delete mode 100644 docs/en/docs/query-acceleration/hint/joinHint.md
delete mode 100644 docs/en/docs/query-acceleration/join-optimization/bucket-shuffle-join.md
delete mode 100644 docs/en/docs/query-acceleration/join-optimization/colocation-join.md
delete mode 100644 docs/en/docs/query-acceleration/join-optimization/doris-join-optimization.md
delete mode 100644 docs/en/docs/query-acceleration/join-optimization/runtime-filter.md
delete mode 100644 docs/en/docs/query-acceleration/materialized-view.md
delete mode 100644 docs/en/docs/query-acceleration/nereids.md
delete mode 100644 docs/en/docs/query-acceleration/pipeline-execution-engine.md
delete mode 100644 docs/en/docs/query-acceleration/pipeline-x-execution-engine.md
delete mode 100644 docs/en/docs/query-acceleration/statistics.md
delete mode 100644 docs/en/docs/releasenotes/release-1.1.0.md
delete mode 100644 docs/en/docs/releasenotes/release-1.1.1.md
delete mode 100644 docs/en/docs/releasenotes/release-1.1.2.md
delete mode 100644 docs/en/docs/releasenotes/release-1.1.3.md
delete mode 100644 docs/en/docs/releasenotes/release-1.1.4.md
delete mode 100644 docs/en/docs/releasenotes/release-1.1.5.md
delete mode 100644 docs/en/docs/releasenotes/release-1.2.0.md
delete mode 100644 docs/en/docs/releasenotes/release-1.2.1.md
delete mode 100644 docs/en/docs/releasenotes/release-1.2.2.md
delete mode 100644 docs/en/docs/releasenotes/release-1.2.3.md
delete mode 100644 docs/en/docs/releasenotes/release-1.2.4.md
delete mode 100644 docs/en/docs/releasenotes/release-1.2.5.md
delete mode 100644 docs/en/docs/releasenotes/release-1.2.6.md
delete mode 100644 docs/en/docs/releasenotes/release-1.2.7.md
delete mode 100644 docs/en/docs/releasenotes/release-1.2.8.md
delete mode 100644 docs/en/docs/releasenotes/release-2.0-Alpha.md
delete mode 100644 docs/en/docs/releasenotes/release-2.0-Beta.md
delete mode 100644 docs/en/docs/releasenotes/release-2.0.0.md
delete mode 100644 docs/en/docs/releasenotes/release-2.0.1.md
delete mode 100644 docs/en/docs/releasenotes/release-2.0.2.md
delete mode 100644 docs/en/docs/releasenotes/release-2.0.3.md
delete mode 100644 docs/en/docs/releasenotes/release-2.0.4.md
delete mode 100644 docs/en/docs/releasenotes/release-2.0.5.md
delete mode 100644 docs/en/docs/releasenotes/release-2.0.6.md
delete mode 100644 docs/en/docs/releasenotes/release-2.1.0.md
create mode 100644 docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-EXTERNAL-TABLE.md
delete mode 100644 docs/sidebars.json
delete mode 100644 docs/sidebarsCommunity.json
delete mode 100644 docs/zh-CN/community/design/doris_storage_optimization.md
delete mode 100644 docs/zh-CN/community/design/grouping_sets_design.md
delete mode 100644 docs/zh-CN/community/design/metadata-design.md
delete mode 100644 docs/zh-CN/community/design/spark_load.md
delete mode 100644 docs/zh-CN/community/developer-guide/be-clion-dev.md
delete mode 100644 docs/zh-CN/community/developer-guide/be-vscode-dev.md
delete mode 100644 docs/zh-CN/community/developer-guide/benchmark-tool.md
delete mode 100644 docs/zh-CN/community/developer-guide/bitmap-hll-file-format.md
delete mode 100644 docs/zh-CN/community/developer-guide/cpp-diagnostic-code.md
delete mode 100644 docs/zh-CN/community/developer-guide/cpp-format-code.md
delete mode 100644 docs/zh-CN/community/developer-guide/debug-tool.md
delete mode 100644 docs/zh-CN/community/developer-guide/docker-dev.md
delete mode 100644 docs/zh-CN/community/developer-guide/fe-eclipse-dev.md
delete mode 100644 docs/zh-CN/community/developer-guide/fe-idea-dev.md
delete mode 100644 docs/zh-CN/community/developer-guide/fe-vscode-dev.md
delete mode 100644 docs/zh-CN/community/developer-guide/github-checks.md
delete mode 100644 docs/zh-CN/community/developer-guide/how-to-share-blogs.md
delete mode 100644 docs/zh-CN/community/developer-guide/java-format-code.md
delete mode 100644 docs/zh-CN/community/developer-guide/mac-dev/be-clion-dev.md
delete mode 100644 docs/zh-CN/community/developer-guide/mac-dev/dev-prepare.md
delete mode 100644 docs/zh-CN/community/developer-guide/mac-dev/fe-idea-dev.md
delete mode 100644 docs/zh-CN/community/developer-guide/regression-testing.md
delete mode 100644 docs/zh-CN/community/doris-future.md
delete mode 100644 docs/zh-CN/community/feedback.md
delete mode 100644 docs/zh-CN/community/how-to-contribute/commit-format-specification.md
delete mode 100644 docs/zh-CN/community/how-to-contribute/contribute-doc.md
delete mode 100644 docs/zh-CN/community/how-to-contribute/contributor-guide.md
delete mode 100644 docs/zh-CN/community/how-to-contribute/how-to-be-a-committer.md
delete mode 100644 docs/zh-CN/community/how-to-contribute/how-to-contribute.md
delete mode 100644 docs/zh-CN/community/how-to-contribute/pull-request.md
delete mode 100644 docs/zh-CN/community/join-community.md
delete mode 100644 docs/zh-CN/community/release-and-verify/release-complete.md
delete mode 100644 docs/zh-CN/community/release-and-verify/release-doris-connectors.md
delete mode 100644 docs/zh-CN/community/release-and-verify/release-doris-core.md
delete mode 100644 docs/zh-CN/community/release-and-verify/release-doris-sdk.md
delete mode 100644 docs/zh-CN/community/release-and-verify/release-doris-shade.md
delete mode 100644 docs/zh-CN/community/release-and-verify/release-prepare.md
delete mode 100644 docs/zh-CN/community/release-and-verify/release-verify.md
delete mode 100644 docs/zh-CN/community/security.md
delete mode 100644 docs/zh-CN/community/subscribe-mail-list.md
delete mode 100644 docs/zh-CN/community/team.md
delete mode 100644 docs/zh-CN/docs/admin-manual/certificate.md
delete mode 100644 docs/zh-CN/docs/admin-manual/cluster-management/elastic-expansion.md
delete mode 100644 docs/zh-CN/docs/admin-manual/cluster-management/fqdn.md
delete mode 100644 docs/zh-CN/docs/admin-manual/cluster-management/load-balancing.md
delete mode 100644 docs/zh-CN/docs/admin-manual/cluster-management/upgrade.md
delete mode 100644 docs/zh-CN/docs/admin-manual/config/be-config.md
delete mode 100644 docs/zh-CN/docs/admin-manual/config/config-dir.md
delete mode 100644 docs/zh-CN/docs/admin-manual/config/fe-config-template.md
delete mode 100644 docs/zh-CN/docs/admin-manual/config/fe-config.md
delete mode 100644 docs/zh-CN/docs/admin-manual/config/user-property.md
delete mode 100644 docs/zh-CN/docs/admin-manual/data-admin/backup.md
delete mode 100644 docs/zh-CN/docs/admin-manual/data-admin/data-recovery.md
delete mode 100644 docs/zh-CN/docs/admin-manual/data-admin/delete-recover.md
delete mode 100644 docs/zh-CN/docs/admin-manual/data-admin/restore.md
delete mode 100644 docs/zh-CN/docs/admin-manual/fe-certificate.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/be/check-rpc-channel.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/be/check-tablet-segment.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/be/checksum.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/be/compaction-run.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/be/compaction-status.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/be/config.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/be/download.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/be/health.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/be/meta.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/be/metrics.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/be/pad-rowset.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/be/reset-rpc-channel.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/be/snapshot.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/be/tablet-distribution.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/be/tablet-info.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/be/tablet-migration.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/be/tablet-reload.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/be/tablet-restore.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/be/version-info.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/backends-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/bootstrap-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/cancel-load-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/check-decommission-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/check-storage-type-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/cluster-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/colocate-meta-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/config-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/connection-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/debug-point-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/extra-basepath-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/fe-version-info-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/get-ddl-stmt-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/get-load-info-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/get-load-state.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/get-log-file-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/get-small-file.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/ha-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/hardware-info-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/health-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/help-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/import-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/log-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/login-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/logout-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/meta-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/meta-info-action-V2.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/meta-info-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/meta-replay-state-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/metrics-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/node-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/profile-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/query-detail-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/query-profile-action-controller.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/query-profile-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/query-schema-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/query-stats-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/row-count-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/session-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/set-config-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/show-data-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/show-meta-info-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/show-proc-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/show-runtime-info-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/show-table-data-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/statement-execution-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/statistic-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/system-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/table-query-plan-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/table-row-count-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/table-schema-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/upload-action.md
delete mode 100644 docs/zh-CN/docs/admin-manual/maint-monitor/automatic-service-start.md
delete mode 100644 docs/zh-CN/docs/admin-manual/maint-monitor/be-olap-error-code.md
delete mode 100644 docs/zh-CN/docs/admin-manual/maint-monitor/disk-capacity.md
delete mode 100644 docs/zh-CN/docs/admin-manual/maint-monitor/doris-error-code.md
delete mode 100644 docs/zh-CN/docs/admin-manual/maint-monitor/memory-management/be-oom-analysis.md
delete mode 100644 docs/zh-CN/docs/admin-manual/maint-monitor/memory-management/memory-limit-exceeded-analysis.md
delete mode 100644 docs/zh-CN/docs/admin-manual/maint-monitor/memory-management/memory-tracker.md
delete mode 100644 docs/zh-CN/docs/admin-manual/maint-monitor/metadata-operation.md
delete mode 100644 docs/zh-CN/docs/admin-manual/maint-monitor/monitor-alert.md
delete mode 100644 docs/zh-CN/docs/admin-manual/maint-monitor/monitor-metrics/metrics.md
delete mode 100644 docs/zh-CN/docs/admin-manual/maint-monitor/tablet-local-debug.md
delete mode 100644 docs/zh-CN/docs/admin-manual/maint-monitor/tablet-meta-tool.md
delete mode 100644 docs/zh-CN/docs/admin-manual/maint-monitor/tablet-repair-and-balance.md
delete mode 100644 docs/zh-CN/docs/admin-manual/multi-tenant.md
delete mode 100644 docs/zh-CN/docs/admin-manual/privilege-ldap/ldap.md
delete mode 100644 docs/zh-CN/docs/admin-manual/privilege-ldap/ranger.md
delete mode 100644 docs/zh-CN/docs/admin-manual/privilege-ldap/user-privilege.md
delete mode 100644 docs/zh-CN/docs/admin-manual/query-profile.md
delete mode 100644 docs/zh-CN/docs/admin-manual/sql-interception.md
delete mode 100644 docs/zh-CN/docs/admin-manual/system-table/rowsets.md
delete mode 100644 docs/zh-CN/docs/admin-manual/workload-group.md
delete mode 100644 docs/zh-CN/docs/advanced/alter-table/replace-table.md
delete mode 100644 docs/zh-CN/docs/advanced/alter-table/schema-change.md
delete mode 100644 docs/zh-CN/docs/advanced/auto-increment.md
delete mode 100644 docs/zh-CN/docs/advanced/autobucket.md
delete mode 100644 docs/zh-CN/docs/advanced/best-practice/compaction.md
delete mode 100644 docs/zh-CN/docs/advanced/best-practice/debug-log.md
delete mode 100644 docs/zh-CN/docs/advanced/best-practice/import-analysis.md
delete mode 100644 docs/zh-CN/docs/advanced/best-practice/query-analysis.md
delete mode 100644 docs/zh-CN/docs/advanced/broker.md
delete mode 100644 docs/zh-CN/docs/advanced/cache/partition-cache-manual.md
delete mode 100644 docs/zh-CN/docs/advanced/cache/query-cache.md
delete mode 100644 docs/zh-CN/docs/advanced/cache/sql-cache-manual.md
delete mode 100644 docs/zh-CN/docs/advanced/cold-hot-separation.md
delete mode 100644 docs/zh-CN/docs/advanced/is-being-synced.md
delete mode 100644 docs/zh-CN/docs/advanced/lateral-view.md
delete mode 100644 docs/zh-CN/docs/advanced/orthogonal-bitmap-manual.md
delete mode 100644 docs/zh-CN/docs/advanced/partition/auto-partition.md
delete mode 100644 docs/zh-CN/docs/advanced/partition/dynamic-partition.md
delete mode 100644 docs/zh-CN/docs/advanced/partition/table-temp-partition.md
delete mode 100644 docs/zh-CN/docs/advanced/resource.md
delete mode 100644 docs/zh-CN/docs/advanced/small-file-mgr.md
delete mode 100644 docs/zh-CN/docs/advanced/sql-mode.md
delete mode 100644 docs/zh-CN/docs/advanced/time-zone.md
delete mode 100644 docs/zh-CN/docs/advanced/user-defined-variables.md
delete mode 100644 docs/zh-CN/docs/advanced/using-hll.md
delete mode 100644 docs/zh-CN/docs/advanced/variables-template.md
delete mode 100644 docs/zh-CN/docs/advanced/variables.md
delete mode 100644 docs/zh-CN/docs/benchmark/ssb.md
delete mode 100644 docs/zh-CN/docs/benchmark/tpch.md
delete mode 100644 docs/zh-CN/docs/data-operate/export/export-manual.md
delete mode 100644 docs/zh-CN/docs/data-operate/export/export-with-mysql-dump.md
delete mode 100644 docs/zh-CN/docs/data-operate/export/outfile.md
delete mode 100644 docs/zh-CN/docs/data-operate/import/import-advanced/min-load-replica-num.md
delete mode 100644 docs/zh-CN/docs/data-operate/import/import-scenes/external-storage-load.md
delete mode 100644 docs/zh-CN/docs/data-operate/import/import-scenes/external-table-load.md
delete mode 100644 docs/zh-CN/docs/data-operate/import/import-scenes/jdbc-load.md
delete mode 100644 docs/zh-CN/docs/data-operate/import/import-scenes/kafka-load.md
delete mode 100644 docs/zh-CN/docs/data-operate/import/import-scenes/load-atomicity.md
delete mode 100644 docs/zh-CN/docs/data-operate/import/import-scenes/load-data-convert.md
delete mode 100644 docs/zh-CN/docs/data-operate/import/import-scenes/load-strict-mode.md
delete mode 100644 docs/zh-CN/docs/data-operate/import/import-scenes/local-file-load.md
delete mode 100644 docs/zh-CN/docs/data-operate/import/import-way/binlog-load-manual.md
delete mode 100644 docs/zh-CN/docs/data-operate/import/import-way/broker-load-manual.md
delete mode 100644 docs/zh-CN/docs/data-operate/import/import-way/group-commit-manual.md
delete mode 100644 docs/zh-CN/docs/data-operate/import/import-way/insert-into-manual.md
delete mode 100644 docs/zh-CN/docs/data-operate/import/import-way/load-json-format.md
delete mode 100644 docs/zh-CN/docs/data-operate/import/import-way/mysql-load-manual.md
delete mode 100644 docs/zh-CN/docs/data-operate/import/import-way/routine-load-manual.md
delete mode 100644 docs/zh-CN/docs/data-operate/import/import-way/s3-load-manual.md
delete mode 100644 docs/zh-CN/docs/data-operate/import/import-way/spark-load-manual.md
delete mode 100644 docs/zh-CN/docs/data-operate/import/import-way/stream-load-manual.md
delete mode 100644 docs/zh-CN/docs/data-operate/import/load-manual.md
delete mode 100644 docs/zh-CN/docs/data-operate/update-delete/batch-delete-manual.md
delete mode 100644 docs/zh-CN/docs/data-operate/update-delete/delete-manual.md
delete mode 100644 docs/zh-CN/docs/data-operate/update-delete/partial-update.md
delete mode 100644 docs/zh-CN/docs/data-operate/update-delete/sequence-column-manual.md
delete mode 100644 docs/zh-CN/docs/data-operate/update-delete/update.md
delete mode 100644 docs/zh-CN/docs/data-table/basic-usage.md
delete mode 100644 docs/zh-CN/docs/data-table/best-practice.md
delete mode 100644 docs/zh-CN/docs/data-table/data-model.md
delete mode 100644 docs/zh-CN/docs/data-table/data-partition.md
delete mode 100644 docs/zh-CN/docs/data-table/hit-the-rollup.md
delete mode 100644 docs/zh-CN/docs/data-table/index/bloomfilter.md
delete mode 100644 docs/zh-CN/docs/data-table/index/index-overview.md
delete mode 100644 docs/zh-CN/docs/data-table/index/inverted-index.md
delete mode 100644 docs/zh-CN/docs/data-table/index/ngram-bloomfilter-index.md
delete mode 100644 docs/zh-CN/docs/ecosystem/audit-plugin.md
delete mode 100644 docs/zh-CN/docs/ecosystem/beats.md
delete mode 100644 docs/zh-CN/docs/ecosystem/bi-integration/apache-superset.md
delete mode 100644 docs/zh-CN/docs/ecosystem/cloudcanal.md
delete mode 100644 docs/zh-CN/docs/ecosystem/datax.md
delete mode 100644 docs/zh-CN/docs/ecosystem/dbt-doris-adapter.md
delete mode 100644 docs/zh-CN/docs/ecosystem/doris-kafka-connector.md
delete mode 100644 docs/zh-CN/docs/ecosystem/doris-streamloader.md
delete mode 100644 docs/zh-CN/docs/ecosystem/flink-doris-connector.md
delete mode 100644 docs/zh-CN/docs/ecosystem/hive-bitmap-udf.md
delete mode 100644 docs/zh-CN/docs/ecosystem/kyuubi.md
delete mode 100644 docs/zh-CN/docs/ecosystem/logstash.md
delete mode 100644 docs/zh-CN/docs/ecosystem/plugin-development-manual.md
delete mode 100644 docs/zh-CN/docs/ecosystem/seatunnel.md
delete mode 100644 docs/zh-CN/docs/ecosystem/spark-doris-connector.md
delete mode 100644 docs/zh-CN/docs/ecosystem/udaf/remote-user-defined-aggregation-function.md
delete mode 100644 docs/zh-CN/docs/ecosystem/udf/contribute-udf.md
delete mode 100644 docs/zh-CN/docs/ecosystem/udf/java-user-defined-function.md
delete mode 100644 docs/zh-CN/docs/ecosystem/udf/remote-user-defined-function.md
delete mode 100644 docs/zh-CN/docs/faq/data-faq.md
delete mode 100644 docs/zh-CN/docs/faq/install-faq.md
delete mode 100644 docs/zh-CN/docs/faq/sql-faq.md
delete mode 100644 docs/zh-CN/docs/get-starting/quick-start.md
delete mode 100644 docs/zh-CN/docs/get-starting/what-is-apache-doris.md
delete mode 100644 docs/zh-CN/docs/install/construct-docker/build-docker-image.md
delete mode 100644 docs/zh-CN/docs/install/construct-docker/run-docker-cluster.md
delete mode 100644 docs/zh-CN/docs/install/k8s-deploy/debug-crash.md
delete mode 100644 docs/zh-CN/docs/install/k8s-deploy/doris-cluster-upgrade.md
delete mode 100644 docs/zh-CN/docs/install/k8s-deploy/expansion-and-contraction.md
delete mode 100644 docs/zh-CN/docs/install/k8s-deploy/helm-chart-deploy.md
delete mode 100644 docs/zh-CN/docs/install/k8s-deploy/network.md
delete mode 100644 docs/zh-CN/docs/install/k8s-deploy/operator-deploy.md
delete mode 100644 docs/zh-CN/docs/install/k8s-deploy/persistent-volume.md
delete mode 100644 docs/zh-CN/docs/install/k8s-deploy/root-user-use.md
delete mode 100644 docs/zh-CN/docs/install/source-install/compilation-arm.md
delete mode 100644 docs/zh-CN/docs/install/source-install/compilation-general.md
delete mode 100644 docs/zh-CN/docs/install/source-install/compilation-mac.md
delete mode 100644 docs/zh-CN/docs/install/source-install/compilation-win.md
delete mode 100644 docs/zh-CN/docs/install/source-install/compilation-with-ldb-toolchain.md
delete mode 100644 docs/zh-CN/docs/install/standard-deployment.md
delete mode 100644 docs/zh-CN/docs/lakehouse/cloud-auth/cloud-auth.md
delete mode 100644 docs/zh-CN/docs/lakehouse/compute-node.md
delete mode 100644 docs/zh-CN/docs/lakehouse/external-statistics.md
delete mode 100644 docs/zh-CN/docs/lakehouse/faq.md
delete mode 100644 docs/zh-CN/docs/lakehouse/file.md
delete mode 100644 docs/zh-CN/docs/lakehouse/filecache.md
delete mode 100644 docs/zh-CN/docs/lakehouse/fs-benchmark-tool.md
delete mode 100644 docs/zh-CN/docs/lakehouse/multi-catalog/dlf.md
delete mode 100644 docs/zh-CN/docs/lakehouse/multi-catalog/es.md
delete mode 100644 docs/zh-CN/docs/lakehouse/multi-catalog/hive.md
delete mode 100644 docs/zh-CN/docs/lakehouse/multi-catalog/hudi.md
delete mode 100644 docs/zh-CN/docs/lakehouse/multi-catalog/iceberg.md
delete mode 100644 docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md
delete mode 100644 docs/zh-CN/docs/lakehouse/multi-catalog/max-compute.md
delete mode 100644 docs/zh-CN/docs/lakehouse/multi-catalog/multi-catalog.md
delete mode 100644 docs/zh-CN/docs/lakehouse/multi-catalog/paimon.md
delete mode 100644 docs/zh-CN/docs/lakehouse/sql-dialect.md
delete mode 100644 docs/zh-CN/docs/query-acceleration/async-materialized-view/async-materialized-view.md
delete mode 100644 docs/zh-CN/docs/query-acceleration/async-materialized-view/query-async-materialized-view.md
delete mode 100644 docs/zh-CN/docs/query-acceleration/hight-concurrent-point-query.md
delete mode 100644 docs/zh-CN/docs/query-acceleration/hint/joinHint.md
delete mode 100644 docs/zh-CN/docs/query-acceleration/join-optimization/bucket-shuffle-join.md
delete mode 100644 docs/zh-CN/docs/query-acceleration/join-optimization/colocation-join.md
delete mode 100644 docs/zh-CN/docs/query-acceleration/join-optimization/doris-join-optimization.md
delete mode 100644 docs/zh-CN/docs/query-acceleration/join-optimization/runtime-filter.md
delete mode 100644 docs/zh-CN/docs/query-acceleration/materialized-view.md
delete mode 100644 docs/zh-CN/docs/query-acceleration/nereids.md
delete mode 100644 docs/zh-CN/docs/query-acceleration/pipeline-execution-engine.md
delete mode 100644 docs/zh-CN/docs/query-acceleration/pipeline-x-execution-engine.md
delete mode 100644 docs/zh-CN/docs/query-acceleration/statistics.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-1.1.0.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-1.1.1.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-1.1.2.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-1.1.3.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-1.1.4.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-1.1.5.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-1.2.0.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-1.2.1.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-1.2.2.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-1.2.3.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-1.2.4.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-1.2.5.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-1.2.6.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-1.2.7.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-1.2.8.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-2.0-Alpha.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-2.0-Beta.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-2.0.0.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-2.0.1.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-2.0.2.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-2.0.3.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-2.0.4.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-2.0.5.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-2.0.6.md
delete mode 100644 docs/zh-CN/docs/releasenotes/release-2.1.0.md
create mode 100644 docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-EXTERNAL-TABLE.md
create mode 100644 fe/be-java-extensions/hudi-scanner/src/main/scala/org/apache/doris/hudi/MORIncrementalSplitReader.scala
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/TableScanParams.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/cloud/load/CloudCopyJob.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/cloud/load/CloudLoadManager.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/cloud/load/CloudRoutineLoadManager.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/cloud/planner/CloudGroupCommitPlanner.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/profile/AggregatedProfile.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCommitter.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveColumnStatistics.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCommonStatistics.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HivePartitionStatistics.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HivePartitionWithStatistics.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/COWIncrementalRelation.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/IncrementalRelation.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/MORIncrementalRelation.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/mysql/BytesChannel.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/mysql/ProxyProtocolHandler.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/DataMaskPolicy.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RangerDataMaskPolicy.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RangerRowFilterPolicy.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RowFilterPolicy.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundHiveTableSink.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSinkCreator.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecTableSinkHashPartitioned.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecTableSinkRandomPartitioned.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ConvertAggStateCast.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/WorkloadGroups.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertCommandContext.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/UnboundLogicalSink.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/tablefunction/WorkloadGroupsTableValuedFunction.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HmsCommitTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/qe/ProxyProtocolHandlerTest.java
create mode 100644 regression-test/data/correctness_p0/test_current_date.out
create mode 100644 regression-test/data/correctness_p0/test_grouping_sets_empty.out
copy regression-test/data/{fault_injection_p0 => inverted_index_p0/storage_format}/documents-1000.json (100%)
create mode 100644 regression-test/data/inverted_index_p0/storage_format/test_storage_format_v1.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query1.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query10.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query11.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query12.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query13.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query14.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query15.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query16.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query17.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query18.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query19.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query2.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query20.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query21.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query22.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query23.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query24.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query25.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query26.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query27.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query28.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query29.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query3.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query30.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query31.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query32.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query33.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query34.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query35.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query36.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query37.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query38.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query39.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query4.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query40.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query41.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query42.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query43.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query5.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query6.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query7.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query8.out
create mode 100644 regression-test/data/nereids_clickbench_shape_p0/query9.out
create mode 100644 regression-test/data/nereids_rules_p0/eliminate_gby_key/eliminate_gby_key.out
create mode 100644 regression-test/data/nereids_rules_p0/grouping_sets/grouping_with_same_grouping_set.out
create mode 100644 regression-test/data/nereids_rules_p0/grouping_sets/remove_duplicate_expr_in_grouping_set.out
create mode 100644 regression-test/data/variant_p0/view.out
create mode 100644 regression-test/plugins/plugin_cluster.groovy
create mode 100644 regression-test/suites/correctness_p0/test_current_date.groovy
create mode 100644 regression-test/suites/correctness_p0/test_grouping_sets_empty.groovy
create mode 100644 regression-test/suites/inverted_index_p0/storage_format/test_storage_format_v1.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/load.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query1.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query10.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query11.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query12.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query13.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query14.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query15.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query16.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query17.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query18.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query19.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query2.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query20.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query21.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query22.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query23.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query24.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query25.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query26.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query27.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query28.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query29.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query3.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query30.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query31.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query32.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query33.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query34.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query35.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query36.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query37.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query38.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query39.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query4.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query40.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query41.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query42.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query43.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query5.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query6.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query7.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query8.groovy
create mode 100644 regression-test/suites/nereids_clickbench_shape_p0/query9.groovy
create mode 100644 regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_gby_key.groovy
create mode 100644 regression-test/suites/nereids_rules_p0/grouping_sets/grouping_with_same_grouping_set.groovy
create mode 100644 regression-test/suites/nereids_rules_p0/grouping_sets/remove_duplicate_expr_in_grouping_set.groovy
create mode 100644 regression-test/suites/variant_p0/view.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 03/09: Support column level health value. (#31794)
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch high-priority-column
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 6fc7aa15ad515f2119f6e399d5cf0398ab395554
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Tue Mar 5 16:19:08 2024 +0800
Support column level health value. (#31794)
---
.../apache/doris/analysis/ShowColumnStatsStmt.java | 4 +
.../apache/doris/datasource/InternalCatalog.java | 6 +-
.../java/org/apache/doris/qe/SessionVariable.java | 7 ++
.../org/apache/doris/statistics/AnalysisInfo.java | 17 ++--
.../doris/statistics/AnalysisInfoBuilder.java | 17 ++--
.../org/apache/doris/statistics/AnalysisJob.java | 8 +-
.../apache/doris/statistics/AnalysisManager.java | 13 ++-
.../org/apache/doris/statistics/ColStatsMeta.java | 16 ++--
.../doris/statistics/StatisticsAutoCollector.java | 95 ++++++++++++++++++++--
.../doris/statistics/StatisticsJobAppender.java | 28 +++++--
.../apache/doris/statistics/TableStatsMeta.java | 33 ++++----
.../doris/statistics/util/StatisticsUtil.java | 10 +++
.../apache/doris/statistics/AnalysisJobTest.java | 4 +-
.../doris/statistics/AnalysisManagerTest.java | 2 +-
14 files changed, 197 insertions(+), 63 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java
index 37be76b20df..749bfa7d360 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java
@@ -61,6 +61,8 @@ public class ShowColumnStatsStmt extends ShowStmt {
.add("trigger")
.add("query_times")
.add("updated_time")
+ .add("update_rows")
+ .add("last_analyze_row_count")
.build();
private final TableName tableName;
@@ -160,6 +162,8 @@ public class ShowColumnStatsStmt extends ShowStmt {
row.add(String.valueOf(colStatsMeta == null ? "N/A" : colStatsMeta.jobType));
row.add(String.valueOf(colStatsMeta == null ? "N/A" : colStatsMeta.queriedTimes));
row.add(String.valueOf(p.second.updatedTime));
+ row.add(String.valueOf(colStatsMeta == null ? "N/A" : colStatsMeta.updatedRows));
+ row.add(String.valueOf(colStatsMeta == null ? "N/A" : colStatsMeta.rowCount));
result.add(row);
});
return new ShowResultSet(getMetaData(), result);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 42a350163f6..d6bbfeeba06 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -3149,7 +3149,6 @@ public class InternalCatalog implements CatalogIf<Database> {
rowsToTruncate += partition.getBaseIndex().getRowCount();
}
} else {
- rowsToTruncate = olapTable.getRowCount();
for (Partition partition : olapTable.getPartitions()) {
// If need absolutely correct, should check running txn here.
// But if the txn is in prepare state, cann't known which partitions had load data.
@@ -3158,6 +3157,7 @@ public class InternalCatalog implements CatalogIf<Database> {
}
origPartitions.put(partition.getName(), partition.getId());
partitionsDistributionInfo.put(partition.getId(), partition.getDistributionInfo());
+ rowsToTruncate += partition.getBaseIndex().getRowCount();
}
}
// if table currently has no partitions, this sql like empty command and do nothing, should return directly.
@@ -3318,10 +3318,8 @@ public class InternalCatalog implements CatalogIf<Database> {
if (truncateEntireTable) {
// Drop the whole table stats after truncate the entire table
Env.getCurrentEnv().getAnalysisManager().dropStats(olapTable);
- } else {
- // Update the updated rows in table stats after truncate some partitions.
- Env.getCurrentEnv().getAnalysisManager().updateUpdatedRows(updateRecords);
}
+ Env.getCurrentEnv().getAnalysisManager().updateUpdatedRows(updateRecords);
LOG.info("finished to truncate table {}, partitions: {}", tblRef.getName().toSql(), tblRef.getPartitionNames());
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 8bfd0d7f1a4..4b8b89ff6a1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -471,6 +471,8 @@ public class SessionVariable implements Serializable, Writable {
public static final String ENABLE_AUTO_ANALYZE = "enable_auto_analyze";
+ public static final String ENABLE_AUTO_ANALYZE_INTERNAL_CATALOG = "enable_auto_analyze_internal_catalog";
+
public static final String AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD = "auto_analyze_table_width_threshold";
public static final String FASTER_FLOAT_CONVERT = "faster_float_convert";
@@ -1504,6 +1506,11 @@ public class SessionVariable implements Serializable, Writable {
flag = VariableMgr.GLOBAL)
public boolean enableAutoAnalyze = true;
+ @VariableMgr.VarAttr(name = ENABLE_AUTO_ANALYZE_INTERNAL_CATALOG,
+ description = {"临时参数，收否自动收集所有内表", "Temp variable， enable to auto collect all OlapTable."},
+ flag = VariableMgr.GLOBAL)
+ public boolean enableAutoAnalyzeInternalCatalog = false;
+
@VariableMgr.VarAttr(name = AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD,
description = {"参与自动收集的最大表宽度，列数多于这个参数的表不参与自动收集",
"Maximum table width to enable auto analyze, "
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
index c707107e0e0..0c5047a53c5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
@@ -188,8 +188,11 @@ public class AnalysisInfo implements Writable {
@SerializedName("endTime")
public long endTime;
- @SerializedName("emptyJob")
- public final boolean emptyJob;
+ @SerializedName("rowCount")
+ public final long rowCount;
+
+ @SerializedName("updateRows")
+ public final long updateRows;
/**
*
* Used to store the newest partition version of tbl when creating this job.
@@ -206,7 +209,8 @@ public class AnalysisInfo implements Writable {
long lastExecTimeInMs, long timeCostInMs, AnalysisState state, ScheduleType scheduleType,
boolean isExternalTableLevelTask, boolean partitionOnly, boolean samplingPartition,
boolean isAllPartition, long partitionCount, CronExpression cronExpression, boolean forceFull,
- boolean usingSqlForPartitionColumn, long tblUpdateTime, boolean emptyJob, boolean userInject) {
+ boolean usingSqlForPartitionColumn, long tblUpdateTime, long rowCount, boolean userInject,
+ long updateRows) {
this.jobId = jobId;
this.taskId = taskId;
this.taskIds = taskIds;
@@ -242,8 +246,9 @@ public class AnalysisInfo implements Writable {
this.forceFull = forceFull;
this.usingSqlForPartitionColumn = usingSqlForPartitionColumn;
this.tblUpdateTime = tblUpdateTime;
- this.emptyJob = emptyJob;
+ this.rowCount = rowCount;
this.userInject = userInject;
+ this.updateRows = updateRows;
}
@Override
@@ -285,7 +290,9 @@ public class AnalysisInfo implements Writable {
}
sj.add("forceFull: " + forceFull);
sj.add("usingSqlForPartitionColumn: " + usingSqlForPartitionColumn);
- sj.add("emptyJob: " + emptyJob);
+ sj.add("rowCount: " + rowCount);
+ sj.add("userInject: " + userInject);
+ sj.add("updateRows: " + updateRows);
return sj.toString();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
index 22f3d22b3ce..527d503fd52 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
@@ -62,8 +62,9 @@ public class AnalysisInfoBuilder {
private boolean forceFull;
private boolean usingSqlForPartitionColumn;
private long tblUpdateTime;
- private boolean emptyJob;
+ private long rowCount;
private boolean userInject;
+ private long updateRows;
public AnalysisInfoBuilder() {
}
@@ -101,8 +102,9 @@ public class AnalysisInfoBuilder {
forceFull = info.forceFull;
usingSqlForPartitionColumn = info.usingSqlForPartitionColumn;
tblUpdateTime = info.tblUpdateTime;
- emptyJob = info.emptyJob;
+ rowCount = info.rowCount;
userInject = info.userInject;
+ updateRows = info.updateRows;
}
public AnalysisInfoBuilder setJobId(long jobId) {
@@ -265,8 +267,8 @@ public class AnalysisInfoBuilder {
return this;
}
- public AnalysisInfoBuilder setEmptyJob(boolean emptyJob) {
- this.emptyJob = emptyJob;
+ public AnalysisInfoBuilder setRowCount(long rowCount) {
+ this.rowCount = rowCount;
return this;
}
@@ -275,12 +277,17 @@ public class AnalysisInfoBuilder {
return this;
}
+ public AnalysisInfoBuilder setUpdateRows(long updateRows) {
+ this.updateRows = updateRows;
+ return this;
+ }
+
public AnalysisInfo build() {
return new AnalysisInfo(jobId, taskId, taskIds, catalogId, dbId, tblId, colToPartitions, partitionNames,
colName, indexId, jobType, analysisMode, analysisMethod, analysisType, samplePercent,
sampleRows, maxBucketNum, periodTimeInMs, message, lastExecTimeInMs, timeCostInMs, state, scheduleType,
externalTableLevelTask, partitionOnly, samplingPartition, isAllPartition, partitionCount,
- cronExpression, forceFull, usingSqlForPartitionColumn, tblUpdateTime, emptyJob, userInject);
+ cronExpression, forceFull, usingSqlForPartitionColumn, tblUpdateTime, rowCount, userInject, updateRows);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java
index f52764bd6c9..600d95a6278 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java
@@ -84,14 +84,12 @@ public class AnalysisJob {
protected void markOneTaskDone() {
if (queryingTask.isEmpty()) {
try {
- writeBuf();
- updateTaskState(AnalysisState.FINISHED, "Cost time in sec: "
- + (System.currentTimeMillis() - start) / 1000);
+ flushBuffer();
} finally {
deregisterJob();
}
} else if (buf.size() >= StatisticsUtil.getInsertMergeCount()) {
- writeBuf();
+ flushBuffer();
}
}
@@ -115,7 +113,7 @@ public class AnalysisJob {
}
}
- protected void writeBuf() {
+ protected void flushBuffer() {
if (killed) {
return;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index ab883c79f8f..bac84996dd1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -416,7 +416,10 @@ public class AnalysisManager implements Writable {
infoBuilder.setColToPartitions(colToPartitions);
infoBuilder.setTaskIds(Lists.newArrayList());
infoBuilder.setTblUpdateTime(table.getUpdateTime());
- infoBuilder.setEmptyJob(table instanceof OlapTable && table.getRowCount() == 0);
+ long rowCount = table.getRowCount();
+ infoBuilder.setRowCount(rowCount);
+ TableStatsMeta tableStatsStatus = findTableStatsStatus(table.getId());
+ infoBuilder.setUpdateRows(tableStatsStatus == null ? 0 : tableStatsStatus.updatedRows.get());
return infoBuilder.build();
}
@@ -572,7 +575,7 @@ public class AnalysisManager implements Writable {
}
TableStatsMeta tableStats = findTableStatsStatus(tbl.getId());
if (tableStats == null) {
- updateTableStatsStatus(new TableStatsMeta(jobInfo.emptyJob ? 0 : tbl.getRowCount(), jobInfo, tbl));
+ updateTableStatsStatus(new TableStatsMeta(jobInfo.rowCount, jobInfo, tbl));
} else {
tableStats.update(jobInfo, tbl);
logCreateTableStats(tableStats);
@@ -807,7 +810,7 @@ public class AnalysisManager implements Writable {
analysisInfo.dbId, analysisInfo.tblId);
return table.createAnalysisTask(analysisInfo);
} catch (Throwable t) {
- LOG.warn("Failed to find table", t);
+ LOG.warn("Failed to create task.", t);
throw new DdlException("Failed to create task", t);
}
}
@@ -1156,10 +1159,12 @@ public class AnalysisManager implements Writable {
public void updateColumnUsedInPredicate(Set<Slot> slotReferences) {
+ LOG.info("Add slots to high priority queues.");
updateColumn(slotReferences, highPriorityColumns);
}
public void updateQueriedColumn(Collection<Slot> slotReferences) {
+ LOG.info("Add slots to mid priority queues.");
updateColumn(slotReferences, midPriorityColumns);
}
@@ -1179,6 +1184,8 @@ public class AnalysisManager implements Writable {
if (catalog != null) {
queue.offer(new HighPriorityColumn(catalog.getId(), database.getId(),
table.getId(), optionalColumn.get().getName()));
+ LOG.info("Offer column " + table.getName() + "(" + table.getId() + ")."
+ + optionalColumn.get().getName());
}
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsMeta.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsMeta.java
index 445641b2505..7e317d67bd7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsMeta.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsMeta.java
@@ -43,16 +43,20 @@ public class ColStatsMeta {
@SerializedName("trigger")
public JobType jobType;
- public ColStatsMeta(long updatedTime, AnalysisMethod analysisMethod,
- AnalysisType analysisType, JobType jobType, long queriedTimes) {
+ @SerializedName("updatedRows")
+ public long updatedRows;
+
+ @SerializedName("rowCount")
+ public long rowCount;
+
+ public ColStatsMeta(long updatedTime, AnalysisMethod analysisMethod, AnalysisType analysisType, JobType jobType,
+ long queriedTimes, long rowCount, long updatedRows) {
this.updatedTime = updatedTime;
this.analysisMethod = analysisMethod;
this.analysisType = analysisType;
this.jobType = jobType;
this.queriedTimes.addAndGet(queriedTimes);
- }
-
- public void clear() {
- updatedTime = 0;
+ this.updatedRows = updatedRows;
+ this.rowCount = rowCount;
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
index 19b9c69db08..c498881bfbf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -23,6 +23,8 @@ import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.util.TimeUtils;
+import org.apache.doris.datasource.hive.HMSExternalTable;
+import org.apache.doris.datasource.hive.HMSExternalTable.DLAType;
import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod;
import org.apache.doris.statistics.AnalysisInfo.JobType;
import org.apache.doris.statistics.AnalysisInfo.ScheduleType;
@@ -33,6 +35,8 @@ import org.apache.logging.log4j.Logger;
import java.time.LocalTime;
import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
@@ -60,9 +64,16 @@ public class StatisticsAutoCollector extends StatisticsCollector {
}
try {
TableIf table = job.getKey();
+ if (!supportAutoAnalyze(table)) {
+ continue;
+ }
Set<String> columns = job.getValue()
.stream()
- .filter(c -> needAnalyzeColumn(table, c))
+ .filter(c -> {
+ boolean needAnalyzeColumn = needAnalyzeColumn(table, c);
+ LOG.info("Need analyze column " + c + " ? " + needAnalyzeColumn);
+ return needAnalyzeColumn;
+ })
.collect(Collectors.toSet());
processOneJob(table, columns);
} catch (Exception e) {
@@ -100,22 +111,92 @@ public class StatisticsAutoCollector extends StatisticsCollector {
}
protected void processOneJob(TableIf table, Set<String> columns) throws DdlException {
- Set<String> collect = columns.stream().filter(c -> needAnalyzeColumn(table, c)).collect(Collectors.toSet());
- if (collect.isEmpty()) {
+ appendPartitionColumns(table, columns);
+ if (columns.isEmpty()) {
return;
}
AnalysisInfo analyzeJob = createAnalyzeJobForTbl(table, columns);
+ LOG.info("Analyze job : {}", analyzeJob.toString());
createSystemAnalysisJob(analyzeJob);
}
+ protected void appendPartitionColumns(TableIf table, Set<String> columns) {
+ if (!(table instanceof OlapTable)) {
+ return;
+ }
+ AnalysisManager manager = Env.getServingEnv().getAnalysisManager();
+ TableStatsMeta tableStatsStatus = manager.findTableStatsStatus(table.getId());
+ if (tableStatsStatus != null && tableStatsStatus.newPartitionLoaded.get()) {
+ OlapTable olapTable = (OlapTable) table;
+ columns.addAll(olapTable.getPartitionNames());
+ }
+ }
+
protected boolean needAnalyzeColumn(TableIf table, String column) {
- //TODO: Calculate column health value.
- return true;
+ AnalysisManager manager = Env.getServingEnv().getAnalysisManager();
+ TableStatsMeta tableStatsStatus = manager.findTableStatsStatus(table.getId());
+ if (tableStatsStatus == null) {
+ return true;
+ }
+ if (tableStatsStatus.userInjected) {
+ return false;
+ }
+ ColStatsMeta columnStatsMeta = tableStatsStatus.findColumnStatsMeta(column);
+ if (columnStatsMeta == null) {
+ return true;
+ }
+ if (table instanceof OlapTable) {
+ long currentUpdatedRows = tableStatsStatus.updatedRows.get();
+ long lastAnalyzeUpdateRows = columnStatsMeta.updatedRows;
+ if (lastAnalyzeUpdateRows == 0 && currentUpdatedRows > 0) {
+ return true;
+ }
+ OlapTable olapTable = (OlapTable) table;
+ if (tableStatsStatus.newPartitionLoaded.get() && olapTable.isPartitionColumn(column)) {
+ return true;
+ }
+ if (columnStatsMeta.rowCount == 0 && olapTable.getRowCount() > 0) {
+ return true;
+ }
+ if (currentUpdatedRows == lastAnalyzeUpdateRows) {
+ return false;
+ }
+ double healthValue = ((double) (currentUpdatedRows - lastAnalyzeUpdateRows)
+ / (double) currentUpdatedRows) * 100.0;
+ LOG.info("Column " + column + " health value is " + healthValue);
+ return healthValue < StatisticsUtil.getTableStatsHealthThreshold();
+ } else {
+ if (!(table instanceof HMSExternalTable)) {
+ return false;
+ }
+ HMSExternalTable hmsTable = (HMSExternalTable) table;
+ if (!hmsTable.getDlaType().equals(DLAType.HIVE)) {
+ return false;
+ }
+ return System.currentTimeMillis()
+ - tableStatsStatus.updatedTime > StatisticsUtil.getExternalTableAutoAnalyzeIntervalInMillis();
+ }
+ }
+
+ protected boolean supportAutoAnalyze(TableIf tableIf) {
+ if (tableIf == null) {
+ return false;
+ }
+ return tableIf instanceof OlapTable
+ || tableIf instanceof HMSExternalTable
+ && ((HMSExternalTable) tableIf).getDlaType().equals(HMSExternalTable.DLAType.HIVE);
}
protected AnalysisInfo createAnalyzeJobForTbl(TableIf table, Set<String> columns) {
AnalysisMethod analysisMethod = table.getDataSize(true) >= StatisticsUtil.getHugeTableLowerBoundSizeInBytes()
? AnalysisMethod.SAMPLE : AnalysisMethod.FULL;
+ AnalysisManager manager = Env.getServingEnv().getAnalysisManager();
+ TableStatsMeta tableStatsStatus = manager.findTableStatsStatus(table.getId());
+ long rowCount = table.getRowCount();
+ Map<String, Set<String>> colToPartitions = new HashMap<>();
+ Set<String> dummyPartition = new HashSet<>();
+ dummyPartition.add("dummy partition");
+ columns.stream().forEach(c -> colToPartitions.put(c, dummyPartition));
return new AnalysisInfoBuilder()
.setJobId(Env.getCurrentEnv().getNextId())
.setCatalogId(table.getDatabase().getCatalog().getId())
@@ -133,7 +214,9 @@ public class StatisticsAutoCollector extends StatisticsCollector {
.setLastExecTimeInMs(System.currentTimeMillis())
.setJobType(JobType.SYSTEM)
.setTblUpdateTime(table.getUpdateTime())
- .setEmptyJob(table instanceof OlapTable && table.getRowCount() == 0)
+ .setRowCount(rowCount)
+ .setUpdateRows(tableStatsStatus == null ? 0 : tableStatsStatus.updatedRows.get())
+ .setColToPartitions(colToPartitions)
.build();
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java
index 73d0d1340ad..71bb71d3cda 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java
@@ -19,6 +19,7 @@ package org.apache.doris.statistics;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.util.MasterDaemon;
@@ -28,7 +29,7 @@ import org.apache.doris.statistics.util.StatisticsUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -70,34 +71,44 @@ public class StatisticsJobAppender extends MasterDaemon {
protected void appendJobs() {
AnalysisManager manager = Env.getCurrentEnv().getAnalysisManager();
+ // LOG.info("Append column to high priority job map.");
appendColumnsToJobs(manager.highPriorityColumns, manager.highPriorityJobs);
+ // LOG.info("Append column to mid priority job map.");
appendColumnsToJobs(manager.midPriorityColumns, manager.midPriorityJobs);
- appendToLowQueue(manager.lowPriorityJobs);
+ if (StatisticsUtil.enableAutoAnalyzeInternalCatalog()) {
+ // LOG.info("Append column to low priority job map.");
+ appendToLowQueue(manager.lowPriorityJobs);
+ }
}
protected void appendColumnsToJobs(Queue<HighPriorityColumn> columnQueue, Map<TableIf, Set<String>> jobsMap) {
int size = columnQueue.size();
for (int i = 0; i < size; i++) {
HighPriorityColumn column = columnQueue.poll();
+ LOG.info("Process column " + column.tblId + "." + column.colName);
TableIf table = StatisticsUtil.findTable(column.catalogId, column.dbId, column.tblId);
synchronized (jobsMap) {
// If job map reach the upper limit, stop putting new jobs.
if (!jobsMap.containsKey(table) && jobsMap.size() >= JOB_MAP_SIZE) {
+ LOG.info("Job map full.");
break;
}
if (jobsMap.containsKey(table)) {
jobsMap.get(table).add(column.colName);
} else {
- jobsMap.put(table, Collections.singleton(column.colName));
+ HashSet<String> columns = new HashSet<>();
+ columns.add(column.colName);
+ jobsMap.put(table, columns);
}
+ LOG.info("Column " + column.tblId + "." + column.colName + " added");
}
}
}
- protected void appendToLowQueue(Map<TableIf, Set<String>> jobsMap) {
-
+ protected void appendToLowQueue(Map<TableIf, Set<String>> jobsMap) {
InternalCatalog catalog = Env.getCurrentInternalCatalog();
List<Long> sortedDbs = catalog.getDbIds().stream().sorted().collect(Collectors.toList());
+ int batchSize = 100;
for (long dbId : sortedDbs) {
if (dbId < currentDbId
|| StatisticConstants.SYSTEM_DBS.contains(catalog.getDbNullable(dbId).getFullName())) {
@@ -108,11 +119,11 @@ public class StatisticsJobAppender extends MasterDaemon {
List<Table> tables = db.get().getTables().stream()
.sorted((t1, t2) -> (int) (t1.getId() - t2.getId())).collect(Collectors.toList());
for (Table t : tables) {
- if (t.getId() <= currentTableId) {
+ if (!(t instanceof OlapTable) || t.getId() <= currentTableId) {
continue;
}
synchronized (jobsMap) {
- // If job map reach the upper limit, stop putting new jobs.
+ // If job map reach the upper limit, stop adding new jobs.
if (!jobsMap.containsKey(t) && jobsMap.size() >= JOB_MAP_SIZE) {
return;
}
@@ -126,6 +137,9 @@ public class StatisticsJobAppender extends MasterDaemon {
}
}
currentTableId = t.getId();
+ if (--batchSize <= 0) {
+ return;
+ }
}
}
// All tables have been processed once, reset for the next loop.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
index 9231c6a2bc7..96ca0aab54c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
@@ -126,11 +126,6 @@ public class TableStatsMeta implements Writable {
return colNameToColStatsMeta.keySet();
}
- public void reset() {
- updatedTime = 0;
- colNameToColStatsMeta.values().forEach(ColStatsMeta::clear);
- }
-
public void update(AnalysisInfo analyzedJob, TableIf tableIf) {
updatedTime = analyzedJob.tblUpdateTime;
userInjected = analyzedJob.userInject;
@@ -145,34 +140,34 @@ public class TableStatsMeta implements Writable {
for (String col : cols) {
ColStatsMeta colStatsMeta = colNameToColStatsMeta.get(col);
if (colStatsMeta == null) {
- colNameToColStatsMeta.put(col, new ColStatsMeta(updatedTime,
- analyzedJob.analysisMethod, analyzedJob.analysisType, analyzedJob.jobType, 0));
+ colNameToColStatsMeta.put(col, new ColStatsMeta(updatedTime, analyzedJob.analysisMethod,
+ analyzedJob.analysisType, analyzedJob.jobType, 0, analyzedJob.rowCount,
+ analyzedJob.updateRows));
} else {
colStatsMeta.updatedTime = updatedTime;
colStatsMeta.analysisType = analyzedJob.analysisType;
colStatsMeta.analysisMethod = analyzedJob.analysisMethod;
colStatsMeta.jobType = analyzedJob.jobType;
+ colStatsMeta.updatedRows = analyzedJob.updateRows;
+ colStatsMeta.rowCount = analyzedJob.rowCount;
}
}
jobType = analyzedJob.jobType;
if (tableIf != null) {
if (tableIf instanceof OlapTable) {
- rowCount = analyzedJob.emptyJob ? 0 : tableIf.getRowCount();
- }
- if (!analyzedJob.emptyJob && analyzedJob.colToPartitions.keySet()
- .containsAll(tableIf.getBaseSchema().stream()
- .filter(c -> !StatisticsUtil.isUnsupportedType(c.getType()))
- .map(Column::getName).collect(Collectors.toSet()))) {
- updatedRows.set(0);
- newPartitionLoaded.set(false);
- }
- if (tableIf instanceof OlapTable) {
+ rowCount = analyzedJob.rowCount;
PartitionInfo partitionInfo = ((OlapTable) tableIf).getPartitionInfo();
- if (partitionInfo != null && analyzedJob.colToPartitions.keySet()
+ if (analyzedJob.rowCount != 0 && partitionInfo != null && analyzedJob.colToPartitions.keySet()
.containsAll(partitionInfo.getPartitionColumns().stream()
- .map(Column::getName).collect(Collectors.toSet()))) {
+ .map(Column::getName).collect(Collectors.toSet()))) {
newPartitionLoaded.set(false);
}
+ if (analyzedJob.rowCount != 0 && analyzedJob.colToPartitions.keySet()
+ .containsAll(tableIf.getBaseSchema().stream()
+ .filter(c -> !StatisticsUtil.isUnsupportedType(c.getType()))
+ .map(Column::getName).collect(Collectors.toSet()))) {
+ userInjected = false;
+ }
}
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index f56aa0db607..0a56a11d115 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -898,6 +898,16 @@ public class StatisticsUtil {
return false;
}
+ public static boolean enableAutoAnalyzeInternalCatalog() {
+ try {
+ return findConfigFromGlobalSessionVar(
+ SessionVariable.ENABLE_AUTO_ANALYZE_INTERNAL_CATALOG).enableAutoAnalyzeInternalCatalog;
+ } catch (Exception e) {
+ LOG.warn("Fail to get value of enable auto analyze internal catalog, return false by default", e);
+ }
+ return false;
+ }
+
public static int getInsertMergeCount() {
try {
return findConfigFromGlobalSessionVar(SessionVariable.STATS_INSERT_MERGE_ITEM_COUNT)
diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java
index 1bf2041bb4f..cb2637d5cf6 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java
@@ -184,7 +184,7 @@ public class AnalysisJobTest {
protected void syncLoadStats() {
}
};
- job.writeBuf();
+ job.flushBuffer();
Assertions.assertEquals(0, job.queryFinished.size());
}
@@ -210,7 +210,7 @@ public class AnalysisJobTest {
job.buf.add(new ColStatsData());
job.queryFinished = new HashSet<>();
job.queryFinished.add(task2);
- job.writeBuf();
+ job.flushBuffer();
Assertions.assertEquals(0, job.queryFinished.size());
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
index f8a77fe06db..4e8bbfe5aff 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
@@ -306,7 +306,7 @@ public class AnalysisManagerTest {
Assertions.assertFalse(olapTable.needReAnalyzeTable(stats2));
TableStatsMeta stats3 = new TableStatsMeta(0, new AnalysisInfoBuilder()
- .setColToPartitions(new HashMap<>()).setEmptyJob(true).setColName("col1").build(), olapTable);
+ .setColToPartitions(new HashMap<>()).setRowCount(0).setColName("col1").build(), olapTable);
Assertions.assertTrue(olapTable.needReAnalyzeTable(stats3));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 07/09: support window (#32094)
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch high-priority-column
in repository https://gitbox.apache.org/repos/asf/doris.git
commit f74a4d07d83ac44df013bb75df8dd6d5f6f42701
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Tue Mar 12 13:25:23 2024 +0800
support window (#32094)
---
.../rules/expression/HighPriorityColumnCollector.java | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/HighPriorityColumnCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/HighPriorityColumnCollector.java
index ed67ad97005..e20363b1202 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/HighPriorityColumnCollector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/HighPriorityColumnCollector.java
@@ -34,6 +34,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalHaving;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.logical.LogicalWindow;
import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
import org.apache.doris.qe.ConnectContext;
@@ -169,6 +170,18 @@ public class HighPriorityColumnCollector extends DefaultPlanRewriter<CollectorCo
return filter;
}
+ @Override
+ public Plan visitLogicalWindow(LogicalWindow<? extends Plan> window, CollectorContext context) {
+ window.child(0).accept(this, context);
+ context.usedInPredicate.addAll(window
+ .getWindowExpressions()
+ .stream()
+ .flatMap(e -> e.<Set<SlotReference>>collect(n -> n instanceof SlotReference).stream())
+ .flatMap(s -> backtrace(s, context).stream())
+ .collect(Collectors.toSet()));
+ return window;
+ }
+
private Set<Slot> backtrace(Slot slot, CollectorContext context) {
return backtrace(slot, new HashSet<>(), context);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 08/09: Refactor. (#32273)
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch high-priority-column
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 7bcec7100f99b1ce4f6aba932ebff51bab83dd5e
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Fri Mar 15 10:14:30 2024 +0800
Refactor. (#32273)
---
.../main/java/org/apache/doris/common/Config.java | 2 +-
.../java/org/apache/doris/catalog/OlapTable.java | 53 ---------------
.../main/java/org/apache/doris/catalog/Table.java | 11 ----
.../java/org/apache/doris/catalog/TableIf.java | 5 --
.../org/apache/doris/datasource/ExternalTable.java | 31 ---------
.../apache/doris/statistics/AnalysisManager.java | 76 +++++-----------------
.../apache/doris/statistics/BaseAnalysisTask.java | 2 +-
.../doris/statistics/ExternalAnalysisTask.java | 4 +-
.../doris/statistics/FollowerColumnSender.java | 4 +-
.../apache/doris/statistics/OlapAnalysisTask.java | 2 +-
.../{HighPriorityColumn.java => QueryColumn.java} | 8 +--
.../doris/statistics/StatisticConstants.java | 2 +-
.../doris/statistics/StatisticsAutoCollector.java | 51 ++++++++++++---
.../doris/statistics/StatisticsCollector.java | 75 ---------------------
.../doris/statistics/StatisticsJobAppender.java | 62 +++++++++++-------
.../doris/statistics/util/StatisticsUtil.java | 54 +++++++--------
.../doris/statistics/AnalysisManagerTest.java | 45 -------------
17 files changed, 137 insertions(+), 350 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index daa9c8b1d35..414dcd0e999 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1602,7 +1602,7 @@ public class Config extends ConfigBase {
"This parameter controls the time interval for automatic collection jobs to check the health of table"
+ "statistics and trigger automatic collection"
})
- public static int auto_check_statistics_in_minutes = 5;
+ public static int auto_check_statistics_in_minutes = 1;
/**
* If set to TRUE, the compaction slower replica will be skipped when select get queryable replicas
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 463b3b1f19d..4ac54827cee 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -59,8 +59,6 @@ import org.apache.doris.statistics.AnalysisInfo.AnalysisType;
import org.apache.doris.statistics.BaseAnalysisTask;
import org.apache.doris.statistics.HistogramTask;
import org.apache.doris.statistics.OlapAnalysisTask;
-import org.apache.doris.statistics.TableStatsMeta;
-import org.apache.doris.statistics.util.StatisticsUtil;
import org.apache.doris.system.Backend;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.TColumn;
@@ -1267,57 +1265,6 @@ public class OlapTable extends Table implements MTMVRelatedTableIf {
}
}
- public boolean needReAnalyzeTable(TableStatsMeta tblStats) {
- if (tblStats == null) {
- return true;
- }
- if (!tblStats.analyzeColumns().containsAll(getBaseSchema()
- .stream()
- .filter(c -> !StatisticsUtil.isUnsupportedType(c.getType()))
- .map(Column::getName)
- .collect(Collectors.toSet()))) {
- return true;
- }
- long rowCount = getRowCount();
- if (rowCount > 0 && tblStats.rowCount == 0) {
- return true;
- }
- long updateRows = tblStats.updatedRows.get();
- int tblHealth = StatisticsUtil.getTableHealth(rowCount, updateRows);
- return tblHealth < StatisticsUtil.getTableStatsHealthThreshold();
- }
-
- @Override
- public Map<String, Set<String>> findReAnalyzeNeededPartitions() {
- TableStatsMeta tableStats = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(getId());
- Set<String> allPartitions = getPartitionNames().stream().map(this::getPartition)
- .filter(Partition::hasData).map(Partition::getName).collect(Collectors.toSet());
- if (tableStats == null) {
- Map<String, Set<String>> ret = Maps.newHashMap();
- for (Column col : getSchemaAllIndexes(false)) {
- if (StatisticsUtil.isUnsupportedType(col.getType())) {
- continue;
- }
- ret.put(col.getName(), allPartitions);
- }
- return ret;
- }
- Map<String, Set<String>> colToPart = new HashMap<>();
- for (Column col : getSchemaAllIndexes(false)) {
- if (StatisticsUtil.isUnsupportedType(col.getType())) {
- continue;
- }
- long lastUpdateTime = tableStats.findColumnLastUpdateTime(col.getName());
- Set<String> partitions = getPartitionNames().stream()
- .map(this::getPartition)
- .filter(Partition::hasData)
- .filter(partition -> partition.getVisibleVersionTime() >= lastUpdateTime).map(Partition::getName)
- .collect(Collectors.toSet());
- colToPart.put(col.getName(), partitions);
- }
- return colToPart;
- }
-
@Override
public long fetchRowCount() {
long rowCount = 0;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
index 35f5b14efc5..619f415719b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
@@ -33,7 +33,6 @@ import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.statistics.AnalysisInfo;
import org.apache.doris.statistics.BaseAnalysisTask;
import org.apache.doris.statistics.ColumnStatistic;
-import org.apache.doris.statistics.TableStatsMeta;
import org.apache.doris.thrift.TTableDescriptor;
import com.google.common.base.Preconditions;
@@ -642,16 +641,6 @@ public abstract class Table extends MetaObject implements Writable, TableIf {
public void analyze(String dbName) {}
- @Override
- public boolean needReAnalyzeTable(TableStatsMeta tblStats) {
- return true;
- }
-
- @Override
- public Map<String, Set<String>> findReAnalyzeNeededPartitions() {
- return Collections.emptyMap();
- }
-
@Override
public List<Long> getChunkSizes() {
throw new NotImplementedException("getChunkSized not implemented");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
index 484dd3bb6eb..5e90f5555d5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
@@ -30,7 +30,6 @@ import org.apache.doris.persist.AlterConstraintLog;
import org.apache.doris.statistics.AnalysisInfo;
import org.apache.doris.statistics.BaseAnalysisTask;
import org.apache.doris.statistics.ColumnStatistic;
-import org.apache.doris.statistics.TableStatsMeta;
import org.apache.doris.thrift.TTableDescriptor;
import com.google.common.collect.ImmutableList;
@@ -182,10 +181,6 @@ public interface TableIf {
Optional<ColumnStatistic> getColumnStatistic(String colName);
- boolean needReAnalyzeTable(TableStatsMeta tblStats);
-
- Map<String, Set<String>> findReAnalyzeNeededPartitions();
-
// Get all the chunk sizes of this table. Now, only HMS external table implemented this interface.
// For HMS external table, the return result is a list of all the files' size.
List<Long> getChunkSizes();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
index 7f82d0d3876..aa7e0385914 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
@@ -32,11 +32,8 @@ import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.statistics.AnalysisInfo;
import org.apache.doris.statistics.BaseAnalysisTask;
import org.apache.doris.statistics.ColumnStatistic;
-import org.apache.doris.statistics.TableStatsMeta;
-import org.apache.doris.statistics.util.StatisticsUtil;
import org.apache.doris.thrift.TTableDescriptor;
-import com.google.common.collect.Sets;
import com.google.gson.annotations.SerializedName;
import lombok.Getter;
import org.apache.commons.lang3.NotImplementedException;
@@ -46,12 +43,9 @@ import org.apache.logging.log4j.Logger;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
/**
* External table represent tables that are not self-managed by Doris.
@@ -317,31 +311,6 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable {
objectCreated = false;
}
- @Override
- public boolean needReAnalyzeTable(TableStatsMeta tblStats) {
- if (tblStats == null) {
- return true;
- }
- if (!tblStats.analyzeColumns().containsAll(getBaseSchema()
- .stream()
- .filter(c -> !StatisticsUtil.isUnsupportedType(c.getType()))
- .map(Column::getName)
- .collect(Collectors.toSet()))) {
- return true;
- }
- return System.currentTimeMillis()
- - tblStats.updatedTime > StatisticsUtil.getExternalTableAutoAnalyzeIntervalInMillis();
- }
-
- @Override
- public Map<String, Set<String>> findReAnalyzeNeededPartitions() {
- HashSet<String> partitions = Sets.newHashSet();
- // TODO: Find a way to collect external table partitions that need to be analyzed.
- partitions.add("Dummy Partition");
- return getBaseSchema().stream().filter(c -> !StatisticsUtil.isUnsupportedType(c.getType()))
- .collect(Collectors.toMap(Column::getName, k -> partitions));
- }
-
@Override
public List<Long> getChunkSizes() {
throw new NotImplementedException("getChunkSized not implemented");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index ddcdf459e6e..8a30aec9e60 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -112,8 +112,8 @@ public class AnalysisManager implements Writable {
private static final Logger LOG = LogManager.getLogger(AnalysisManager.class);
private static final int COLUMN_QUEUE_SIZE = 1000;
- public final Queue<HighPriorityColumn> highPriorityColumns = new ArrayBlockingQueue<>(COLUMN_QUEUE_SIZE);
- public final Queue<HighPriorityColumn> midPriorityColumns = new ArrayBlockingQueue<>(COLUMN_QUEUE_SIZE);
+ public final Queue<QueryColumn> highPriorityColumns = new ArrayBlockingQueue<>(COLUMN_QUEUE_SIZE);
+ public final Queue<QueryColumn> midPriorityColumns = new ArrayBlockingQueue<>(COLUMN_QUEUE_SIZE);
public final Map<TableName, Set<String>> highPriorityJobs = new LinkedHashMap<>();
public final Map<TableName, Set<String>> midPriorityJobs = new LinkedHashMap<>();
public final Map<TableName, Set<String>> lowPriorityJobs = new LinkedHashMap<>();
@@ -307,55 +307,10 @@ public class AnalysisManager implements Writable {
}
}
- /**
- * Gets the partitions for which statistics are to be collected. First verify that
- * there are partitions that have been deleted but have historical statistics(invalid statistics),
- * if there are these partitions, we need to delete them to avoid errors in summary table level statistics.
- * Then get the partitions for which statistics need to be collected based on collection mode (incremental/full).
- * <p>
- * note:
- * If there is no invalid statistics, it does not need to collect/update
- * statistics if the following conditions are met:
- * - in full collection mode, the partitioned table does not have partitions
- * - in incremental collection mode, partition statistics already exist
- * <p>
- * TODO Supports incremental collection of statistics from materialized views
- */
- private Map<String, Set<String>> validateAndGetPartitions(TableIf table, Set<String> columnNames,
- Set<String> partitionNames, AnalysisType analysisType) throws DdlException {
-
- Map<String, Set<String>> columnToPartitions = columnNames.stream()
- .collect(Collectors.toMap(
- columnName -> columnName,
- columnName -> new HashSet<>(partitionNames == null ? Collections.emptySet() : partitionNames)
- ));
-
- if (analysisType == AnalysisType.HISTOGRAM) {
- // Collecting histograms does not need to support incremental collection,
- // and will automatically cover historical statistics
- return columnToPartitions;
- }
-
- if (table instanceof HMSExternalTable) {
- // TODO Currently, we do not support INCREMENTAL collection for external table.
- // One reason is external table partition id couldn't convert to a Long value.
- // Will solve this problem later.
- return columnToPartitions;
- }
-
- if (analysisType == AnalysisType.FUNDAMENTALS) {
- Map<String, Set<String>> result = table.findReAnalyzeNeededPartitions();
- result.keySet().retainAll(columnNames);
- return result;
- }
-
- return columnToPartitions;
- }
-
// Make sure colName of job has all the column as this AnalyzeStmt specified, no matter whether it will be analyzed
// or not.
@VisibleForTesting
- public AnalysisInfo buildAnalysisJobInfo(AnalyzeTblStmt stmt) throws DdlException {
+ public AnalysisInfo buildAnalysisJobInfo(AnalyzeTblStmt stmt) {
AnalysisInfoBuilder infoBuilder = new AnalysisInfoBuilder();
long jobId = Env.getCurrentEnv().getNextId();
TableIf table = stmt.getTable();
@@ -413,9 +368,10 @@ public class AnalysisManager implements Writable {
long periodTimeInMs = stmt.getPeriodTimeInMs();
infoBuilder.setPeriodTimeInMs(periodTimeInMs);
-
- Map<String, Set<String>> colToPartitions = validateAndGetPartitions(table, columnNames,
- partitionNames, analysisType);
+ Map<String, Set<String>> colToPartitions = new HashMap<>();
+ Set<String> dummyPartition = new HashSet<>();
+ dummyPartition.add("dummy partition");
+ columnNames.stream().forEach(c -> colToPartitions.put(c, dummyPartition));
infoBuilder.setColToPartitions(colToPartitions);
infoBuilder.setTaskIds(Lists.newArrayList());
infoBuilder.setTblUpdateTime(table.getUpdateTime());
@@ -770,6 +726,7 @@ public class AnalysisManager implements Writable {
}
tableStats.updatedTime = 0;
tableStats.userInjected = false;
+ tableStats.rowCount = table.getRowCount();
}
public void invalidateRemoteStats(long catalogId, long dbId, long tableId,
@@ -1196,16 +1153,14 @@ public class AnalysisManager implements Writable {
public void updateColumnUsedInPredicate(Set<Slot> slotReferences) {
- LOG.info("Add slots to high priority queues.");
updateColumn(slotReferences, highPriorityColumns);
}
public void updateQueriedColumn(Collection<Slot> slotReferences) {
- LOG.info("Add slots to mid priority queues.");
updateColumn(slotReferences, midPriorityColumns);
}
- protected void updateColumn(Collection<Slot> slotReferences, Queue<HighPriorityColumn> queue) {
+ protected void updateColumn(Collection<Slot> slotReferences, Queue<QueryColumn> queue) {
for (Slot s : slotReferences) {
if (!(s instanceof SlotReference)) {
return;
@@ -1219,10 +1174,12 @@ public class AnalysisManager implements Writable {
if (database != null) {
CatalogIf catalog = database.getCatalog();
if (catalog != null) {
- queue.offer(new HighPriorityColumn(catalog.getId(), database.getId(),
+ queue.offer(new QueryColumn(catalog.getId(), database.getId(),
table.getId(), optionalColumn.get().getName()));
- LOG.info("Offer column " + table.getName() + "(" + table.getId() + ")."
- + optionalColumn.get().getName());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Offer column " + table.getName() + "(" + table.getId() + ")."
+ + optionalColumn.get().getName());
+ }
}
}
}
@@ -1231,14 +1188,15 @@ public class AnalysisManager implements Writable {
public void mergeFollowerQueryColumns(Collection<TQueryColumn> highColumns,
Collection<TQueryColumn> midColumns) {
+ LOG.info("Received {} high columns and {} mid columns", highColumns.size(), midColumns.size());
for (TQueryColumn c : highColumns) {
- if (!highPriorityColumns.offer(new HighPriorityColumn(Long.parseLong(c.catalogId), Long.parseLong(c.dbId),
+ if (!highPriorityColumns.offer(new QueryColumn(Long.parseLong(c.catalogId), Long.parseLong(c.dbId),
Long.parseLong(c.tblId), c.colName))) {
break;
}
}
for (TQueryColumn c : midColumns) {
- if (!midPriorityColumns.offer(new HighPriorityColumn(Long.parseLong(c.catalogId), Long.parseLong(c.dbId),
+ if (!midPriorityColumns.offer(new QueryColumn(Long.parseLong(c.catalogId), Long.parseLong(c.dbId),
Long.parseLong(c.tblId), c.colName))) {
break;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
index 68767843507..4e1c3abe944 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
@@ -48,7 +48,7 @@ public abstract class BaseAnalysisTask {
public static final long LIMIT_SIZE = 1024 * 1024 * 1024; // 1GB
public static final double LIMIT_FACTOR = 1.2;
- protected static final String COLLECT_COL_STATISTICS =
+ protected static final String FULL_ANALYZE_TEMPLATE =
"SELECT CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}') AS `id`, "
+ " ${catalogId} AS `catalog_id`, "
+ " ${dbId} AS `db_id`, "
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java
index ef1b795bd13..690aa237439 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java
@@ -73,6 +73,8 @@ public class ExternalAnalysisTask extends BaseAnalysisTask {
*/
private void getTableStats() {
Map<String, String> params = buildStatsParams(null);
+ Pair<Double, Long> sampleInfo = getSampleInfo();
+ params.put("scaleFactor", String.valueOf(sampleInfo.first));
List<ResultRow> columnResult =
StatisticsUtil.execStatisticQuery(new StringSubstitutor(params)
.replace(ANALYZE_TABLE_COUNT_TEMPLATE));
@@ -98,7 +100,7 @@ public class ExternalAnalysisTask extends BaseAnalysisTask {
if (LOG.isDebugEnabled()) {
LOG.debug("Will do full collection for column {}", col.getName());
}
- sb.append(COLLECT_COL_STATISTICS);
+ sb.append(FULL_ANALYZE_TEMPLATE);
} else {
// Do sample analyze
if (LOG.isDebugEnabled()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/FollowerColumnSender.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/FollowerColumnSender.java
index 0a804152694..51ff9501308 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/FollowerColumnSender.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/FollowerColumnSender.java
@@ -74,14 +74,14 @@ public class FollowerColumnSender extends MasterDaemon {
= analysisManager.highPriorityColumns
.stream()
.filter(c -> StatisticsUtil.needAnalyzeColumn(c))
- .map(HighPriorityColumn::toThrift)
+ .map(QueryColumn::toThrift)
.collect(Collectors.toSet());
Set<TQueryColumn> midPriorityColumns
= analysisManager.midPriorityColumns
.stream()
.filter(c -> StatisticsUtil.needAnalyzeColumn(c))
.filter(c -> !highPriorityColumns.contains(c))
- .map(HighPriorityColumn::toThrift)
+ .map(QueryColumn::toThrift)
.collect(Collectors.toSet());
analysisManager.highPriorityColumns.clear();
analysisManager.midPriorityColumns.clear();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
index 0cb33be80e0..29c67784577 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
@@ -228,7 +228,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
params.put("tblName", String.valueOf(tbl.getName()));
params.put("index", getIndex());
StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
- String collectColStats = stringSubstitutor.replace(COLLECT_COL_STATISTICS);
+ String collectColStats = stringSubstitutor.replace(FULL_ANALYZE_TEMPLATE);
runQuery(collectColStats);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HighPriorityColumn.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/QueryColumn.java
similarity index 88%
rename from fe/fe-core/src/main/java/org/apache/doris/statistics/HighPriorityColumn.java
rename to fe/fe-core/src/main/java/org/apache/doris/statistics/QueryColumn.java
index d619ef82c08..df91ea7f4c0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HighPriorityColumn.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/QueryColumn.java
@@ -21,14 +21,14 @@ import org.apache.doris.thrift.TQueryColumn;
import java.util.Objects;
-public class HighPriorityColumn {
+public class QueryColumn {
public final long catalogId;
public final long dbId;
public final long tblId;
public final String colName;
- public HighPriorityColumn(long catalogId, long dbId, long tblId, String colName) {
+ public QueryColumn(long catalogId, long dbId, long tblId, String colName) {
this.catalogId = catalogId;
this.dbId = dbId;
this.tblId = tblId;
@@ -45,10 +45,10 @@ public class HighPriorityColumn {
if (this == other) {
return true;
}
- if (!(other instanceof HighPriorityColumn)) {
+ if (!(other instanceof QueryColumn)) {
return false;
}
- HighPriorityColumn otherCriticalColumn = (HighPriorityColumn) other;
+ QueryColumn otherCriticalColumn = (QueryColumn) other;
return this.catalogId == otherCriticalColumn.catalogId
&& this.dbId == otherCriticalColumn.dbId
&& this.tblId == otherCriticalColumn.tblId
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
index 74c7bd7c9db..314cf6648bd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
@@ -95,7 +95,7 @@ public class StatisticConstants {
public static final int ANALYZE_TIMEOUT_IN_SEC = 43200;
- public static final int TASK_QUEUE_CAP = 10;
+ public static final int TASK_QUEUE_CAP = 1;
public static final int AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD = 100;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
index c26e7b05efd..43ee1af2032 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -24,6 +24,7 @@ import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.Pair;
+import org.apache.doris.common.util.MasterDaemon;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod;
@@ -31,6 +32,7 @@ import org.apache.doris.statistics.AnalysisInfo.JobType;
import org.apache.doris.statistics.AnalysisInfo.ScheduleType;
import org.apache.doris.statistics.util.StatisticsUtil;
+import org.apache.hudi.common.util.VisibleForTesting;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -45,18 +47,33 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
-public class StatisticsAutoCollector extends StatisticsCollector {
+public class StatisticsAutoCollector extends MasterDaemon {
private static final Logger LOG = LogManager.getLogger(StatisticsAutoCollector.class);
+ protected final AnalysisTaskExecutor analysisTaskExecutor;
+
public StatisticsAutoCollector() {
- super("Automatic Analyzer",
- TimeUnit.MINUTES.toMillis(Config.auto_check_statistics_in_minutes),
- new AnalysisTaskExecutor(Config.auto_analyze_simultaneously_running_task_num,
- StatisticConstants.TASK_QUEUE_CAP));
+ super("Automatic Analyzer", TimeUnit.MINUTES.toMillis(Config.auto_check_statistics_in_minutes));
+ this.analysisTaskExecutor = new AnalysisTaskExecutor(Config.auto_analyze_simultaneously_running_task_num,
+ StatisticConstants.TASK_QUEUE_CAP);
}
@Override
+ protected void runAfterCatalogReady() {
+ if (!Env.getCurrentEnv().isMaster()) {
+ return;
+ }
+ if (!StatisticsUtil.statsTblAvailable()) {
+ LOG.info("Stats table not available, skip");
+ return;
+ }
+ if (Env.isCheckpointThread()) {
+ return;
+ }
+ collect();
+ }
+
protected void collect() {
while (canCollect()) {
Pair<Entry<TableName, Set<String>>, JobPriority> job = getJob();
@@ -70,8 +87,7 @@ public class StatisticsAutoCollector extends StatisticsCollector {
if (!supportAutoAnalyze(table)) {
continue;
}
- Set<String> columns = job.first.getValue().stream().collect(Collectors.toSet());
- processOneJob(table, columns, job.second);
+ processOneJob(table, job.first.getValue(), job.second);
} catch (Exception e) {
LOG.warn("Failed to analyze table {} with columns [{}]", job.first.getKey().getTbl(),
job.first.getValue().stream().collect(Collectors.joining(",")), e);
@@ -107,13 +123,14 @@ public class StatisticsAutoCollector extends StatisticsCollector {
}
protected void processOneJob(TableIf table, Set<String> columns, JobPriority priority) throws DdlException {
+ columns = columns.stream().filter(c -> StatisticsUtil.needAnalyzeColumn(table, c)).collect(Collectors.toSet());
appendPartitionColumns(table, columns);
if (columns.isEmpty()) {
return;
}
AnalysisInfo analyzeJob = createAnalyzeJobForTbl(table, columns, priority);
- LOG.info("Analyze job : {}", analyzeJob.toString());
- createSystemAnalysisJob(analyzeJob);
+ LOG.debug("Auto analyze job : {}", analyzeJob.toString());
+ executeSystemAnalysisJob(analyzeJob);
}
protected void appendPartitionColumns(TableIf table, Set<String> columns) {
@@ -170,4 +187,20 @@ public class StatisticsAutoCollector extends StatisticsCollector {
.setPriority(priority)
.build();
}
+
+ // Analysis job created by the system
+ @VisibleForTesting
+ protected void executeSystemAnalysisJob(AnalysisInfo jobInfo)
+ throws DdlException {
+ Map<Long, BaseAnalysisTask> analysisTasks = new HashMap<>();
+ AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
+ analysisManager.createTaskForEachColumns(jobInfo, analysisTasks, false);
+ if (StatisticsUtil.isExternalTable(jobInfo.catalogId, jobInfo.dbId, jobInfo.tblId)
+ && jobInfo.priority.equals(JobPriority.LOW)) {
+ analysisManager.createTableLevelTaskForExternalTable(jobInfo, analysisTasks, false);
+ }
+ Env.getCurrentEnv().getAnalysisManager().constructJob(jobInfo, analysisTasks.values());
+ Env.getCurrentEnv().getAnalysisManager().registerSysJob(jobInfo, analysisTasks);
+ analysisTasks.values().forEach(analysisTaskExecutor::submitTask);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCollector.java
deleted file mode 100644
index e26d3170178..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCollector.java
+++ /dev/null
@@ -1,75 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.statistics;
-
-import org.apache.doris.catalog.Env;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.common.util.MasterDaemon;
-import org.apache.doris.statistics.util.StatisticsUtil;
-
-import org.apache.hudi.common.util.VisibleForTesting;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public abstract class StatisticsCollector extends MasterDaemon {
-
- private static final Logger LOG = LogManager.getLogger(StatisticsCollector.class);
-
- protected final AnalysisTaskExecutor analysisTaskExecutor;
-
- public StatisticsCollector(String name, long intervalMs, AnalysisTaskExecutor analysisTaskExecutor) {
- super(name, intervalMs);
- this.analysisTaskExecutor = analysisTaskExecutor;
- }
-
- @Override
- protected void runAfterCatalogReady() {
- if (!Env.getCurrentEnv().isMaster()) {
- return;
- }
- if (!StatisticsUtil.statsTblAvailable()) {
- LOG.info("Stats table not available, skip");
- return;
- }
- if (Env.isCheckpointThread()) {
- return;
- }
- collect();
- }
-
- protected abstract void collect();
-
- // Analysis job created by the system
- @VisibleForTesting
- protected void createSystemAnalysisJob(AnalysisInfo jobInfo)
- throws DdlException {
- Map<Long, BaseAnalysisTask> analysisTasks = new HashMap<>();
- AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
- analysisManager.createTaskForEachColumns(jobInfo, analysisTasks, false);
- if (StatisticsUtil.isExternalTable(jobInfo.catalogId, jobInfo.dbId, jobInfo.tblId)) {
- analysisManager.createTableLevelTaskForExternalTable(jobInfo, analysisTasks, false);
- }
- Env.getCurrentEnv().getAnalysisManager().constructJob(jobInfo, analysisTasks.values());
- Env.getCurrentEnv().getAnalysisManager().registerSysJob(jobInfo, analysisTasks);
- analysisTasks.values().forEach(analysisTaskExecutor::submitTask);
- }
-
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java
index 9e07c65e2fe..336171d8858 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java
@@ -36,6 +36,7 @@ import java.util.Map;
import java.util.Optional;
import java.util.Queue;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
public class StatisticsJobAppender extends MasterDaemon {
@@ -44,9 +45,12 @@ public class StatisticsJobAppender extends MasterDaemon {
public static final long INTERVAL = 1000;
public static final int JOB_MAP_SIZE = 1000;
+ public static final int TABLE_BATCH_SIZE = 100;
- private long currentDbId;
- private long currentTableId;
+ private long currentDbId = 0;
+ private long currentTableId = 0;
+ private long lastRoundFinishTime = 0;
+ private long lowJobIntervalMs = TimeUnit.MINUTES.toMillis(1);
public StatisticsJobAppender() {
super("Statistics Job Appender", INTERVAL);
@@ -60,10 +64,6 @@ public class StatisticsJobAppender extends MasterDaemon {
if (!Env.getCurrentEnv().isMaster()) {
return;
}
- if (!StatisticsUtil.statsTblAvailable()) {
- LOG.info("Stats table not available, skip");
- return;
- }
if (Env.isCheckpointThread()) {
return;
}
@@ -72,31 +72,28 @@ public class StatisticsJobAppender extends MasterDaemon {
protected void appendJobs() {
AnalysisManager manager = Env.getCurrentEnv().getAnalysisManager();
- // LOG.info("Append column to high priority job map.");
appendColumnsToJobs(manager.highPriorityColumns, manager.highPriorityJobs);
- // LOG.info("Append column to mid priority job map.");
appendColumnsToJobs(manager.midPriorityColumns, manager.midPriorityJobs);
if (StatisticsUtil.enableAutoAnalyzeInternalCatalog()) {
- // LOG.info("Append column to low priority job map.");
- appendToLowQueue(manager.lowPriorityJobs);
+ appendToLowJobs(manager.lowPriorityJobs);
}
}
- protected void appendColumnsToJobs(Queue<HighPriorityColumn> columnQueue, Map<TableName, Set<String>> jobsMap) {
+ protected void appendColumnsToJobs(Queue<QueryColumn> columnQueue, Map<TableName, Set<String>> jobsMap) {
int size = columnQueue.size();
+ int processed = 0;
for (int i = 0; i < size; i++) {
- HighPriorityColumn column = columnQueue.poll();
+ QueryColumn column = columnQueue.poll();
if (!StatisticsUtil.needAnalyzeColumn(column)) {
continue;
}
- LOG.info("Process column " + column.tblId + "." + column.colName);
TableIf table = StatisticsUtil.findTable(column.catalogId, column.dbId, column.tblId);
TableName tableName = new TableName(table.getDatabase().getCatalog().getName(),
table.getDatabase().getFullName(), table.getName());
synchronized (jobsMap) {
// If job map reach the upper limit, stop putting new jobs.
if (!jobsMap.containsKey(tableName) && jobsMap.size() >= JOB_MAP_SIZE) {
- LOG.info("Job map full.");
+ LOG.info("High or mid job map full.");
break;
}
if (jobsMap.containsKey(tableName)) {
@@ -106,15 +103,21 @@ public class StatisticsJobAppender extends MasterDaemon {
columns.add(column.colName);
jobsMap.put(tableName, columns);
}
- LOG.info("Column " + column.tblId + "." + column.colName + " added");
}
+ processed++;
+ }
+ if (size > 0 && LOG.isDebugEnabled()) {
+ LOG.debug("{} of {} columns append to jobs", processed, size);
}
}
- protected void appendToLowQueue(Map<TableName, Set<String>> jobsMap) {
+ protected void appendToLowJobs(Map<TableName, Set<String>> jobsMap) {
+ if (System.currentTimeMillis() - lastRoundFinishTime < lowJobIntervalMs) {
+ return;
+ }
InternalCatalog catalog = Env.getCurrentInternalCatalog();
List<Long> sortedDbs = catalog.getDbIds().stream().sorted().collect(Collectors.toList());
- int batchSize = 100;
+ int processed = 0;
for (long dbId : sortedDbs) {
if (dbId < currentDbId
|| StatisticConstants.SYSTEM_DBS.contains(catalog.getDbNullable(dbId).getFullName())) {
@@ -128,31 +131,40 @@ public class StatisticsJobAppender extends MasterDaemon {
if (!(t instanceof OlapTable) || t.getId() <= currentTableId) {
continue;
}
- TableName tableName = new TableName(t.getDatabase().getCatalog().getName(),
- t.getDatabase().getFullName(), t.getName());
+ OlapTable olapTable = (OlapTable) t;
+ Set<String> columns = olapTable.getColumns().stream()
+ .filter(c -> !StatisticsUtil.isUnsupportedType(c.getType()))
+ .filter(c -> StatisticsUtil.needAnalyzeColumn(olapTable, c.getName()))
+ .map(c -> c.getName()).collect(Collectors.toSet());
+ if (columns.isEmpty()) {
+ continue;
+ }
+ TableName tableName = new TableName(olapTable.getDatabase().getCatalog().getName(),
+ olapTable.getDatabase().getFullName(), olapTable.getName());
synchronized (jobsMap) {
// If job map reach the upper limit, stop adding new jobs.
if (!jobsMap.containsKey(tableName) && jobsMap.size() >= JOB_MAP_SIZE) {
+ LOG.info("Low job map full.");
return;
}
- Set<String> columns = t.getColumns().stream()
- .filter(c -> !StatisticsUtil.isUnsupportedType(c.getType()))
- .filter(c -> StatisticsUtil.needAnalyzeColumn(t, c.getName()))
- .map(c -> c.getName()).collect(Collectors.toSet());
if (jobsMap.containsKey(tableName)) {
jobsMap.get(tableName).addAll(columns);
} else {
jobsMap.put(tableName, columns);
}
}
- currentTableId = t.getId();
- if (--batchSize <= 0) {
+ currentTableId = olapTable.getId();
+ if (++processed > TABLE_BATCH_SIZE) {
return;
}
}
}
// All tables have been processed once, reset for the next loop.
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("All low priority internal tables are appended once.");
+ }
currentDbId = 0;
currentTableId = 0;
+ lastRoundFinishTime = System.currentTimeMillis();
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index 3ce8e7966af..20e98564776 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -75,8 +75,8 @@ import org.apache.doris.statistics.AnalysisManager;
import org.apache.doris.statistics.ColStatsMeta;
import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.ColumnStatisticBuilder;
-import org.apache.doris.statistics.HighPriorityColumn;
import org.apache.doris.statistics.Histogram;
+import org.apache.doris.statistics.QueryColumn;
import org.apache.doris.statistics.ResultRow;
import org.apache.doris.statistics.StatisticConstants;
import org.apache.doris.statistics.TableStatsMeta;
@@ -1044,57 +1044,58 @@ public class StatisticsUtil {
return true;
}
- // TODO: Need refactor, hard to understand now.
public static boolean needAnalyzeColumn(TableIf table, String column) {
AnalysisManager manager = Env.getServingEnv().getAnalysisManager();
TableStatsMeta tableStatsStatus = manager.findTableStatsStatus(table.getId());
+ // Table never been analyzed, need analyze.
if (tableStatsStatus == null) {
return true;
}
+ // User injected column stats, don't do auto analyze, avoid overwrite user injected stats.
if (tableStatsStatus.userInjected) {
return false;
}
ColStatsMeta columnStatsMeta = tableStatsStatus.findColumnStatsMeta(column);
+ // Column never been analyzed, need analyze.
if (columnStatsMeta == null) {
return true;
}
if (table instanceof OlapTable) {
- long currentUpdatedRows = tableStatsStatus.updatedRows.get();
- long lastAnalyzeUpdateRows = columnStatsMeta.updatedRows;
- if (lastAnalyzeUpdateRows == 0 && currentUpdatedRows > 0) {
- return true;
- }
- if (lastAnalyzeUpdateRows > currentUpdatedRows) {
- // Shouldn't happen. Just in case.
+ OlapTable olapTable = (OlapTable) table;
+ // 0. Check new partition first time loaded flag.
+ if (olapTable.isPartitionColumn(column) && tableStatsStatus.newPartitionLoaded.get()) {
return true;
}
- OlapTable olapTable = (OlapTable) table;
+ // 1. Check row count.
+ // TODO: One conner case. Last analyze row count is 0, but actually it's not 0 because isEmptyTable waiting.
long currentRowCount = olapTable.getRowCount();
long lastAnalyzeRowCount = columnStatsMeta.rowCount;
- if (tableStatsStatus.newPartitionLoaded.get() && olapTable.isPartitionColumn(column)) {
+ // 1.1 Empty table -> non-empty table. Need analyze.
+ if (currentRowCount != 0 && lastAnalyzeRowCount == 0) {
return true;
}
- if (lastAnalyzeRowCount == 0 && currentRowCount > 0) {
+ // 1.2 Non-empty table -> empty table. Need analyze;
+ if (currentRowCount == 0 && lastAnalyzeRowCount != 0) {
return true;
}
- if (currentUpdatedRows == lastAnalyzeUpdateRows) {
+ // 1.3 Table is still empty. Not need to analyze. lastAnalyzeRowCount == 0 is always true here.
+ if (currentRowCount == 0) {
return false;
}
- double healthValue = ((double) (currentUpdatedRows - lastAnalyzeUpdateRows)
- / (double) currentUpdatedRows) * 100.0;
- LOG.info("Column " + column + " update rows health value is " + healthValue);
- if (healthValue < StatisticsUtil.getTableStatsHealthThreshold()) {
+ // 1.4 If row count changed more than the threshold, need analyze.
+ // lastAnalyzeRowCount == 0 is always false here.
+ double changeRate =
+ ((double) Math.abs(currentRowCount - lastAnalyzeRowCount) / lastAnalyzeRowCount) * 100.0;
+ if (changeRate > StatisticsUtil.getTableStatsHealthThreshold()) {
return true;
}
- if (currentRowCount == 0 && lastAnalyzeRowCount != 0) {
- return true;
- }
- if (currentRowCount == 0 && lastAnalyzeRowCount == 0) {
- return false;
- }
- healthValue = ((double) (currentRowCount - lastAnalyzeRowCount) / (double) currentRowCount) * 100.0;
- return healthValue < StatisticsUtil.getTableStatsHealthThreshold();
+ // 2. Check update rows.
+ long currentUpdatedRows = tableStatsStatus.updatedRows.get();
+ long lastAnalyzeUpdateRows = columnStatsMeta.updatedRows;
+ changeRate = ((double) Math.abs(currentUpdatedRows - lastAnalyzeUpdateRows) / lastAnalyzeRowCount) * 100.0;
+ return changeRate > StatisticsUtil.getTableStatsHealthThreshold();
} else {
+ // Now, we only support Hive external table auto analyze.
if (!(table instanceof HMSExternalTable)) {
return false;
}
@@ -1102,12 +1103,13 @@ public class StatisticsUtil {
if (!hmsTable.getDlaType().equals(DLAType.HIVE)) {
return false;
}
+ // External is hard to calculate change rate, use time interval to control analyze frequency.
return System.currentTimeMillis()
- tableStatsStatus.updatedTime > StatisticsUtil.getExternalTableAutoAnalyzeIntervalInMillis();
}
}
- public static boolean needAnalyzeColumn(HighPriorityColumn column) {
+ public static boolean needAnalyzeColumn(QueryColumn column) {
if (column == null) {
return false;
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
index 4e8bbfe5aff..53cb0807b80 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
@@ -22,9 +22,6 @@ import org.apache.doris.analysis.AnalyzeTblStmt;
import org.apache.doris.analysis.PartitionNames;
import org.apache.doris.analysis.ShowAnalyzeStmt;
import org.apache.doris.analysis.TableName;
-import org.apache.doris.catalog.Column;
-import org.apache.doris.catalog.OlapTable;
-import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.statistics.AnalysisInfo.AnalysisType;
@@ -33,7 +30,6 @@ import org.apache.doris.statistics.AnalysisInfo.ScheduleType;
import org.apache.doris.statistics.util.StatisticsUtil;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Lists;
import mockit.Expectations;
import mockit.Injectable;
import mockit.Mock;
@@ -270,47 +266,6 @@ public class AnalysisManagerTest {
};
}
- @Test
- public void testReAnalyze() {
- new MockUp<OlapTable>() {
-
- int count = 0;
- int[] rowCount = new int[]{100, 100, 200, 200, 1, 1};
-
- final Column c = new Column("col1", PrimitiveType.INT);
- @Mock
- public long getRowCount() {
- return rowCount[count++];
- }
-
- @Mock
- public List<Column> getBaseSchema() {
- return Lists.newArrayList(c);
- }
-
- @Mock
- public List<Column> getColumns() { return Lists.newArrayList(c); }
-
- };
- OlapTable olapTable = new OlapTable();
- TableStatsMeta stats1 = new TableStatsMeta(
- 50, new AnalysisInfoBuilder().setColToPartitions(new HashMap<>())
- .setColName("col1").build(), olapTable);
- stats1.updatedRows.addAndGet(50);
-
- Assertions.assertTrue(olapTable.needReAnalyzeTable(stats1));
- TableStatsMeta stats2 = new TableStatsMeta(
- 190, new AnalysisInfoBuilder()
- .setColToPartitions(new HashMap<>()).setColName("col1").build(), olapTable);
- stats2.updatedRows.addAndGet(20);
- Assertions.assertFalse(olapTable.needReAnalyzeTable(stats2));
-
- TableStatsMeta stats3 = new TableStatsMeta(0, new AnalysisInfoBuilder()
- .setColToPartitions(new HashMap<>()).setRowCount(0).setColName("col1").build(), olapTable);
- Assertions.assertTrue(olapTable.needReAnalyzeTable(stats3));
-
- }
-
@Test
public void testRecordLimit1() {
Config.analyze_record_limit = 2;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 04/09: Support follower sync query columns to master. (#31859)
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch high-priority-column
in repository https://gitbox.apache.org/repos/asf/doris.git
commit fe58392bd4729cb82efbdaeabd7dd9ab13512cfe
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Wed Mar 6 17:05:21 2024 +0800
Support follower sync query columns to master. (#31859)
---
.../main/java/org/apache/doris/catalog/Env.java | 8 ++
.../apache/doris/service/FrontendServiceImpl.java | 8 ++
.../apache/doris/statistics/AnalysisManager.java | 15 +++
.../doris/statistics/FollowerColumnSender.java | 120 +++++++++++++++++++++
.../doris/statistics/HighPriorityColumn.java | 11 ++
.../doris/statistics/StatisticsAutoCollector.java | 21 +++-
gensrc/thrift/FrontendService.thrift | 13 +++
7 files changed, 194 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index ba31670cbc2..945ccd1be5e 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -240,6 +240,7 @@ import org.apache.doris.scheduler.registry.ExportTaskRegister;
import org.apache.doris.service.ExecuteEnv;
import org.apache.doris.service.FrontendOptions;
import org.apache.doris.statistics.AnalysisManager;
+import org.apache.doris.statistics.FollowerColumnSender;
import org.apache.doris.statistics.StatisticsAutoCollector;
import org.apache.doris.statistics.StatisticsCache;
import org.apache.doris.statistics.StatisticsCleaner;
@@ -522,6 +523,8 @@ public class Env {
private StatisticsJobAppender statisticsJobAppender;
+ private FollowerColumnSender followerColumnSender;
+
private HiveTransactionMgr hiveTransactionMgr;
private TopicPublisherThread topicPublisherThread;
@@ -1732,6 +1735,11 @@ public class Env {
if (analysisManager != null) {
analysisManager.getStatisticsCache().preHeat();
}
+
+ if (followerColumnSender == null) {
+ followerColumnSender = new FollowerColumnSender();
+ followerColumnSender.start();
+ }
}
// Set global variable 'lower_case_table_names' only when the cluster is initialized.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index 04b7ea43997..c6f85fb96aa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -222,6 +222,7 @@ import org.apache.doris.thrift.TStreamLoadMultiTablePutResult;
import org.apache.doris.thrift.TStreamLoadPutRequest;
import org.apache.doris.thrift.TStreamLoadPutResult;
import org.apache.doris.thrift.TStringLiteral;
+import org.apache.doris.thrift.TSyncQueryColumns;
import org.apache.doris.thrift.TTableIndexQueryStats;
import org.apache.doris.thrift.TTableMetadataNameIds;
import org.apache.doris.thrift.TTableQueryStats;
@@ -3815,4 +3816,11 @@ public class FrontendServiceImpl implements FrontendService.Iface {
return result;
}
+ @Override
+ public TStatus syncQueryColumns(TSyncQueryColumns request) throws TException {
+ Env.getCurrentEnv().getAnalysisManager().mergeFollowerQueryColumns(request.highPriorityColumns,
+ request.midPriorityColumns);
+ return new TStatus(TStatusCode.OK);
+ }
+
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index bac84996dd1..e2bd85a0f43 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -64,6 +64,7 @@ import org.apache.doris.statistics.util.StatisticsUtil;
import org.apache.doris.system.Frontend;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.TInvalidateFollowerStatsCacheRequest;
+import org.apache.doris.thrift.TQueryColumn;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
@@ -1191,4 +1192,18 @@ public class AnalysisManager implements Writable {
}
}
}
+
+ public void mergeFollowerQueryColumns(Collection<TQueryColumn> highColumns,
+ Collection<TQueryColumn> midColumns) {
+ for (TQueryColumn c : highColumns) {
+ if (!highPriorityColumns.offer(new HighPriorityColumn(c.catalogId, c.dbId, c.tblId, c.colName))) {
+ break;
+ }
+ }
+ for (TQueryColumn c : midColumns) {
+ if (!midPriorityColumns.offer(new HighPriorityColumn(c.catalogId, c.dbId, c.tblId, c.colName))) {
+ break;
+ }
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/FollowerColumnSender.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/FollowerColumnSender.java
new file mode 100644
index 00000000000..181000c1ef2
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/FollowerColumnSender.java
@@ -0,0 +1,120 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.statistics;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.ClientPool;
+import org.apache.doris.common.util.MasterDaemon;
+import org.apache.doris.ha.FrontendNodeType;
+import org.apache.doris.statistics.util.StatisticsUtil;
+import org.apache.doris.system.Frontend;
+import org.apache.doris.thrift.FrontendService;
+import org.apache.doris.thrift.TNetworkAddress;
+import org.apache.doris.thrift.TQueryColumn;
+import org.apache.doris.thrift.TSyncQueryColumns;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.net.InetSocketAddress;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class FollowerColumnSender extends MasterDaemon {
+
+ private static final Logger LOG = LogManager.getLogger(FollowerColumnSender.class);
+
+ public static final long INTERVAL = 5000;
+
+ public FollowerColumnSender() {
+ super("Follower Column Sender", INTERVAL);
+ }
+
+ @Override
+ protected void runAfterCatalogReady() {
+ if (!StatisticsUtil.enableAutoAnalyze()) {
+ return;
+ }
+ if (Env.getCurrentEnv().isMaster()) {
+ return;
+ }
+ if (Env.isCheckpointThread()) {
+ return;
+ }
+ send();
+ }
+
+ protected void send() {
+ if (Env.getCurrentEnv().isMaster()) {
+ return;
+ }
+ Env currentEnv = Env.getCurrentEnv();
+ AnalysisManager analysisManager = currentEnv.getAnalysisManager();
+ if (analysisManager.highPriorityColumns.isEmpty() && analysisManager.midPriorityColumns.isEmpty()) {
+ return;
+ }
+ List<TQueryColumn> highPriorityColumns
+ = analysisManager.highPriorityColumns
+ .stream()
+ .map(HighPriorityColumn::toThrift)
+ .collect(Collectors.toList());
+ List<TQueryColumn> midPriorityColumns
+ = analysisManager.midPriorityColumns
+ .stream()
+ .map(HighPriorityColumn::toThrift)
+ .collect(Collectors.toList());
+ analysisManager.highPriorityColumns.clear();
+ analysisManager.midPriorityColumns.clear();
+ TSyncQueryColumns queryColumns = new TSyncQueryColumns();
+ queryColumns.highPriorityColumns = highPriorityColumns;
+ queryColumns.midPriorityColumns = midPriorityColumns;
+ Frontend master = null;
+ try {
+ InetSocketAddress masterAddress = currentEnv.getHaProtocol().getLeader();
+ for (Frontend fe : currentEnv.getFrontends(FrontendNodeType.FOLLOWER)) {
+ InetSocketAddress socketAddress = new InetSocketAddress(fe.getHost(), fe.getEditLogPort());
+ if (socketAddress.equals(masterAddress)) {
+ master = fe;
+ break;
+ }
+ }
+ } catch (Exception e) {
+ LOG.warn("Failed to find master FE.", e);
+ return;
+ }
+
+ if (master == null) {
+ LOG.warn("No master found in cluster.");
+ return;
+ }
+ TNetworkAddress address = new TNetworkAddress(master.getHost(), master.getRpcPort());
+ FrontendService.Client client = null;
+ try {
+ client = ClientPool.frontendPool.borrowObject(address);
+ client.syncQueryColumns(queryColumns);
+ LOG.info("Send {} high priority columns and {} mid priority columns to master.",
+ highPriorityColumns.size(), midPriorityColumns.size());
+ } catch (Throwable t) {
+ LOG.warn("Failed to sync stats to master: {}", address, t);
+ } finally {
+ if (client != null) {
+ ClientPool.frontendPool.returnObject(address, client);
+ }
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HighPriorityColumn.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HighPriorityColumn.java
index c4bc20c399a..b2292ef725d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HighPriorityColumn.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HighPriorityColumn.java
@@ -17,6 +17,8 @@
package org.apache.doris.statistics;
+import org.apache.doris.thrift.TQueryColumn;
+
import java.util.Objects;
public class HighPriorityColumn {
@@ -52,4 +54,13 @@ public class HighPriorityColumn {
&& this.tblId == otherCriticalColumn.tblId
&& this.colName.equals(otherCriticalColumn.colName);
}
+
+ public TQueryColumn toThrift() {
+ TQueryColumn tQueryColumn = new TQueryColumn();
+ tQueryColumn.catalogId = catalogId;
+ tQueryColumn.dbId = dbId;
+ tQueryColumn.tblId = tblId;
+ tQueryColumn.colName = colName;
+ return tQueryColumn;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
index c498881bfbf..e0df94b5cb0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -132,6 +132,7 @@ public class StatisticsAutoCollector extends StatisticsCollector {
}
}
+ // TODO: Need refactor, hard to understand now.
protected boolean needAnalyzeColumn(TableIf table, String column) {
AnalysisManager manager = Env.getServingEnv().getAnalysisManager();
TableStatsMeta tableStatsStatus = manager.findTableStatsStatus(table.getId());
@@ -151,11 +152,17 @@ public class StatisticsAutoCollector extends StatisticsCollector {
if (lastAnalyzeUpdateRows == 0 && currentUpdatedRows > 0) {
return true;
}
+ if (lastAnalyzeUpdateRows > currentUpdatedRows) {
+ // Shouldn't happen. Just in case.
+ return true;
+ }
OlapTable olapTable = (OlapTable) table;
+ long currentRowCount = olapTable.getRowCount();
+ long lastAnalyzeRowCount = columnStatsMeta.rowCount;
if (tableStatsStatus.newPartitionLoaded.get() && olapTable.isPartitionColumn(column)) {
return true;
}
- if (columnStatsMeta.rowCount == 0 && olapTable.getRowCount() > 0) {
+ if (lastAnalyzeRowCount == 0 && currentRowCount > 0) {
return true;
}
if (currentUpdatedRows == lastAnalyzeUpdateRows) {
@@ -163,7 +170,17 @@ public class StatisticsAutoCollector extends StatisticsCollector {
}
double healthValue = ((double) (currentUpdatedRows - lastAnalyzeUpdateRows)
/ (double) currentUpdatedRows) * 100.0;
- LOG.info("Column " + column + " health value is " + healthValue);
+ LOG.info("Column " + column + " update rows health value is " + healthValue);
+ if (healthValue < StatisticsUtil.getTableStatsHealthThreshold()) {
+ return true;
+ }
+ if (currentRowCount == 0 && lastAnalyzeRowCount != 0) {
+ return true;
+ }
+ if (currentRowCount == 0 && lastAnalyzeRowCount == 0) {
+ return false;
+ }
+ healthValue = ((double) (currentRowCount - lastAnalyzeRowCount) / (double) currentRowCount) * 100.0;
return healthValue < StatisticsUtil.getTableStatsHealthThreshold();
} else {
if (!(table instanceof HMSExternalTable)) {
diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift
index c7bdbb4fc0e..2ddf52afd49 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -1422,6 +1422,18 @@ struct TReportCommitTxnResultRequest {
4: optional binary payload
}
+struct TQueryColumn {
+ 1: optional i64 catalogId
+ 2: optional i64 dbId
+ 3: optional i64 tblId
+ 4: optional string colName
+}
+
+struct TSyncQueryColumns {
+ 1: optional list<TQueryColumn> highPriorityColumns;
+ 2: optional list<TQueryColumn> midPriorityColumns;
+}
+
service FrontendService {
TGetDbsResult getDbNames(1: TGetDbsParams params)
TGetTablesResult getTableNames(1: TGetTablesParams params)
@@ -1509,4 +1521,5 @@ service FrontendService {
TShowProcessListResult showProcessList(1: TShowProcessListRequest request)
Status.TStatus reportCommitTxnResult(1: TReportCommitTxnResultRequest request)
+ Status.TStatus syncQueryColumns(1: TSyncQueryColumns request)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 06/09: Check column health value earlier, show job priority. (#32064)
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch high-priority-column
in repository https://gitbox.apache.org/repos/asf/doris.git
commit c28bdc7fd758f3a77fb1bb643bb8bd3ada316ac7
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Mon Mar 11 19:46:20 2024 +0800
Check column health value earlier, show job priority. (#32064)
---
.../org/apache/doris/analysis/ShowAnalyzeStmt.java | 1 +
.../java/org/apache/doris/qe/SessionVariable.java | 2 +-
.../java/org/apache/doris/qe/ShowExecutor.java | 4 +-
.../org/apache/doris/statistics/AnalysisInfo.java | 8 +-
.../doris/statistics/AnalysisInfoBuilder.java | 10 +-
.../apache/doris/statistics/AnalysisManager.java | 10 +-
.../doris/statistics/FollowerColumnSender.java | 23 +++--
.../doris/statistics/HighPriorityColumn.java | 6 +-
.../org/apache/doris/statistics/JobPriority.java | 3 +-
.../doris/statistics/StatisticsAutoCollector.java | 102 ++++-----------------
.../doris/statistics/StatisticsJobAppender.java | 8 +-
.../doris/statistics/util/StatisticsUtil.java | 88 +++++++++++++++++-
gensrc/thrift/FrontendService.thrift | 6 +-
13 files changed, 161 insertions(+), 110 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java
index efcfc517024..734073901fe 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java
@@ -62,6 +62,7 @@ public class ShowAnalyzeStmt extends ShowStmt {
.add("schedule_type")
.add("start_time")
.add("end_time")
+ .add("priority")
.build();
private long jobId;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 4b8b89ff6a1..39819faf7d1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -1509,7 +1509,7 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = ENABLE_AUTO_ANALYZE_INTERNAL_CATALOG,
description = {"临时参数，收否自动收集所有内表", "Temp variable， enable to auto collect all OlapTable."},
flag = VariableMgr.GLOBAL)
- public boolean enableAutoAnalyzeInternalCatalog = false;
+ public boolean enableAutoAnalyzeInternalCatalog = true;
@VariableMgr.VarAttr(name = AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD,
description = {"参与自动收集的最大表宽度，列数多于这个参数的表不参与自动收集",
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index 670ec85f6f7..b2bbd1b31b1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -2867,6 +2867,7 @@ public class ShowExecutor {
java.time.ZoneId.systemDefault());
row.add(startTime.format(formatter));
row.add(endTime.format(formatter));
+ row.add(analysisInfo.priority.name());
resultRows.add(row);
} catch (Exception e) {
LOG.warn("Failed to get analyze info for table {}.{}.{}, reason: {}",
@@ -2884,8 +2885,7 @@ public class ShowExecutor {
for (AutoAnalysisPendingJob job : jobs) {
try {
List<String> row = new ArrayList<>();
- CatalogIf<? extends DatabaseIf<? extends TableIf>> c
- = StatisticsUtil.findCatalog(job.catalogName);
+ CatalogIf<? extends DatabaseIf<? extends TableIf>> c = StatisticsUtil.findCatalog(job.catalogName);
row.add(c.getName());
Optional<? extends DatabaseIf<? extends TableIf>> databaseIf = c.getDb(job.dbName);
row.add(databaseIf.isPresent() ? databaseIf.get().getFullName() : "DB may get deleted");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
index 0c5047a53c5..24cf6f38d68 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
@@ -200,8 +200,12 @@ public class AnalysisInfo implements Writable {
*/
public final long tblUpdateTime;
+ @SerializedName("userInject")
public final boolean userInject;
+ @SerializedName("priority")
+ public final JobPriority priority;
+
public AnalysisInfo(long jobId, long taskId, List<Long> taskIds, long catalogId, long dbId, long tblId,
Map<String, Set<String>> colToPartitions, Set<String> partitionNames, String colName, Long indexId,
JobType jobType, AnalysisMode analysisMode, AnalysisMethod analysisMethod, AnalysisType analysisType,
@@ -210,7 +214,7 @@ public class AnalysisInfo implements Writable {
boolean isExternalTableLevelTask, boolean partitionOnly, boolean samplingPartition,
boolean isAllPartition, long partitionCount, CronExpression cronExpression, boolean forceFull,
boolean usingSqlForPartitionColumn, long tblUpdateTime, long rowCount, boolean userInject,
- long updateRows) {
+ long updateRows, JobPriority priority) {
this.jobId = jobId;
this.taskId = taskId;
this.taskIds = taskIds;
@@ -249,6 +253,7 @@ public class AnalysisInfo implements Writable {
this.rowCount = rowCount;
this.userInject = userInject;
this.updateRows = updateRows;
+ this.priority = priority;
}
@Override
@@ -293,6 +298,7 @@ public class AnalysisInfo implements Writable {
sj.add("rowCount: " + rowCount);
sj.add("userInject: " + userInject);
sj.add("updateRows: " + updateRows);
+ sj.add("priority: " + priority.name());
return sj.toString();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
index 527d503fd52..2f60b258598 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
@@ -65,6 +65,7 @@ public class AnalysisInfoBuilder {
private long rowCount;
private boolean userInject;
private long updateRows;
+ private JobPriority priority;
public AnalysisInfoBuilder() {
}
@@ -105,6 +106,7 @@ public class AnalysisInfoBuilder {
rowCount = info.rowCount;
userInject = info.userInject;
updateRows = info.updateRows;
+ priority = info.priority;
}
public AnalysisInfoBuilder setJobId(long jobId) {
@@ -282,12 +284,18 @@ public class AnalysisInfoBuilder {
return this;
}
+ public AnalysisInfoBuilder setPriority(JobPriority priority) {
+ this.priority = priority;
+ return this;
+ }
+
public AnalysisInfo build() {
return new AnalysisInfo(jobId, taskId, taskIds, catalogId, dbId, tblId, colToPartitions, partitionNames,
colName, indexId, jobType, analysisMode, analysisMethod, analysisType, samplePercent,
sampleRows, maxBucketNum, periodTimeInMs, message, lastExecTimeInMs, timeCostInMs, state, scheduleType,
externalTableLevelTask, partitionOnly, samplingPartition, isAllPartition, partitionCount,
- cronExpression, forceFull, usingSqlForPartitionColumn, tblUpdateTime, rowCount, userInject, updateRows);
+ cronExpression, forceFull, usingSqlForPartitionColumn, tblUpdateTime, rowCount, userInject, updateRows,
+ priority);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index c6713b3a7c1..ddcdf459e6e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -219,7 +219,8 @@ public class AnalysisManager implements Writable {
public void createAnalysisJob(AnalyzeTblStmt stmt, boolean proxy) throws DdlException {
// Using auto analyzer if user specifies.
if (stmt.getAnalyzeProperties().getProperties().containsKey("use.auto.analyzer")) {
- Env.getCurrentEnv().getStatisticsAutoCollector().processOneJob(stmt.getTable(), stmt.getColumnNames());
+ Env.getCurrentEnv().getStatisticsAutoCollector()
+ .processOneJob(stmt.getTable(), stmt.getColumnNames(), JobPriority.HIGH);
return;
}
AnalysisInfo jobInfo = buildAndAssignJob(stmt);
@@ -422,6 +423,7 @@ public class AnalysisManager implements Writable {
infoBuilder.setRowCount(rowCount);
TableStatsMeta tableStatsStatus = findTableStatsStatus(table.getId());
infoBuilder.setUpdateRows(tableStatsStatus == null ? 0 : tableStatsStatus.updatedRows.get());
+ infoBuilder.setPriority(JobPriority.MANUAL);
return infoBuilder.build();
}
@@ -1230,12 +1232,14 @@ public class AnalysisManager implements Writable {
public void mergeFollowerQueryColumns(Collection<TQueryColumn> highColumns,
Collection<TQueryColumn> midColumns) {
for (TQueryColumn c : highColumns) {
- if (!highPriorityColumns.offer(new HighPriorityColumn(c.catalogId, c.dbId, c.tblId, c.colName))) {
+ if (!highPriorityColumns.offer(new HighPriorityColumn(Long.parseLong(c.catalogId), Long.parseLong(c.dbId),
+ Long.parseLong(c.tblId), c.colName))) {
break;
}
}
for (TQueryColumn c : midColumns) {
- if (!midPriorityColumns.offer(new HighPriorityColumn(c.catalogId, c.dbId, c.tblId, c.colName))) {
+ if (!midPriorityColumns.offer(new HighPriorityColumn(Long.parseLong(c.catalogId), Long.parseLong(c.dbId),
+ Long.parseLong(c.tblId), c.colName))) {
break;
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/FollowerColumnSender.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/FollowerColumnSender.java
index 181000c1ef2..0a804152694 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/FollowerColumnSender.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/FollowerColumnSender.java
@@ -32,14 +32,16 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.net.InetSocketAddress;
+import java.util.ArrayList;
import java.util.List;
+import java.util.Set;
import java.util.stream.Collectors;
public class FollowerColumnSender extends MasterDaemon {
private static final Logger LOG = LogManager.getLogger(FollowerColumnSender.class);
- public static final long INTERVAL = 5000;
+ public static final long INTERVAL = 60000;
public FollowerColumnSender() {
super("Follower Column Sender", INTERVAL);
@@ -68,21 +70,28 @@ public class FollowerColumnSender extends MasterDaemon {
if (analysisManager.highPriorityColumns.isEmpty() && analysisManager.midPriorityColumns.isEmpty()) {
return;
}
- List<TQueryColumn> highPriorityColumns
+ Set<TQueryColumn> highPriorityColumns
= analysisManager.highPriorityColumns
.stream()
+ .filter(c -> StatisticsUtil.needAnalyzeColumn(c))
.map(HighPriorityColumn::toThrift)
- .collect(Collectors.toList());
- List<TQueryColumn> midPriorityColumns
+ .collect(Collectors.toSet());
+ Set<TQueryColumn> midPriorityColumns
= analysisManager.midPriorityColumns
.stream()
+ .filter(c -> StatisticsUtil.needAnalyzeColumn(c))
+ .filter(c -> !highPriorityColumns.contains(c))
.map(HighPriorityColumn::toThrift)
- .collect(Collectors.toList());
+ .collect(Collectors.toSet());
analysisManager.highPriorityColumns.clear();
analysisManager.midPriorityColumns.clear();
TSyncQueryColumns queryColumns = new TSyncQueryColumns();
- queryColumns.highPriorityColumns = highPriorityColumns;
- queryColumns.midPriorityColumns = midPriorityColumns;
+ List<TQueryColumn> highs = new ArrayList<>();
+ highs.addAll(highPriorityColumns);
+ queryColumns.highPriorityColumns = highs;
+ List<TQueryColumn> mids = new ArrayList<>();
+ mids.addAll(midPriorityColumns);
+ queryColumns.midPriorityColumns = mids;
Frontend master = null;
try {
InetSocketAddress masterAddress = currentEnv.getHaProtocol().getLeader();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HighPriorityColumn.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HighPriorityColumn.java
index b2292ef725d..d619ef82c08 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HighPriorityColumn.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HighPriorityColumn.java
@@ -57,9 +57,9 @@ public class HighPriorityColumn {
public TQueryColumn toThrift() {
TQueryColumn tQueryColumn = new TQueryColumn();
- tQueryColumn.catalogId = catalogId;
- tQueryColumn.dbId = dbId;
- tQueryColumn.tblId = tblId;
+ tQueryColumn.catalogId = String.valueOf(catalogId);
+ tQueryColumn.dbId = String.valueOf(dbId);
+ tQueryColumn.tblId = String.valueOf(tblId);
tQueryColumn.colName = colName;
return tQueryColumn;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/JobPriority.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/JobPriority.java
index 2786b063563..c3656b92927 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/JobPriority.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/JobPriority.java
@@ -20,5 +20,6 @@ package org.apache.doris.statistics;
public enum JobPriority {
HIGH,
MID,
- LOW;
+ LOW,
+ MANUAL;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
index 227074dbb5c..c26e7b05efd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -23,9 +23,9 @@ import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
+import org.apache.doris.common.Pair;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.datasource.hive.HMSExternalTable;
-import org.apache.doris.datasource.hive.HMSExternalTable.DLAType;
import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod;
import org.apache.doris.statistics.AnalysisInfo.JobType;
import org.apache.doris.statistics.AnalysisInfo.ScheduleType;
@@ -39,6 +39,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -58,29 +59,22 @@ public class StatisticsAutoCollector extends StatisticsCollector {
@Override
protected void collect() {
while (canCollect()) {
- Map.Entry<TableName, Set<String>> job = getJob();
+ Pair<Entry<TableName, Set<String>>, JobPriority> job = getJob();
if (job == null) {
// No more job to process, break and sleep.
break;
}
try {
- TableName tblName = job.getKey();
+ TableName tblName = job.first.getKey();
TableIf table = StatisticsUtil.findTable(tblName.getCtl(), tblName.getDb(), tblName.getTbl());
if (!supportAutoAnalyze(table)) {
continue;
}
- Set<String> columns = job.getValue()
- .stream()
- .filter(c -> {
- boolean needAnalyzeColumn = needAnalyzeColumn(table, c);
- LOG.info("Need analyze column " + c + " ? " + needAnalyzeColumn);
- return needAnalyzeColumn;
- })
- .collect(Collectors.toSet());
- processOneJob(table, columns);
+ Set<String> columns = job.first.getValue().stream().collect(Collectors.toSet());
+ processOneJob(table, columns, job.second);
} catch (Exception e) {
- LOG.warn("Failed to analyze table {} with columns [{}]",
- job.getKey().getTbl(), job.getValue().stream().collect(Collectors.joining(",")), e);
+ LOG.warn("Failed to analyze table {} with columns [{}]", job.first.getKey().getTbl(),
+ job.first.getValue().stream().collect(Collectors.joining(",")), e);
}
}
}
@@ -90,18 +84,18 @@ public class StatisticsAutoCollector extends StatisticsCollector {
&& StatisticsUtil.inAnalyzeTime(LocalTime.now(TimeUtils.getTimeZone().toZoneId()));
}
- protected Map.Entry<TableName, Set<String>> getJob() {
+ protected Pair<Entry<TableName, Set<String>>, JobPriority> getJob() {
AnalysisManager manager = Env.getServingEnv().getAnalysisManager();
- Optional<Map.Entry<TableName, Set<String>>> job = fetchJobFromMap(manager.highPriorityJobs);
+ Optional<Entry<TableName, Set<String>>> job = fetchJobFromMap(manager.highPriorityJobs);
if (job.isPresent()) {
- return job.get();
+ return Pair.of(job.get(), JobPriority.HIGH);
}
job = fetchJobFromMap(manager.midPriorityJobs);
if (job.isPresent()) {
- return job.get();
+ return Pair.of(job.get(), JobPriority.MID);
}
job = fetchJobFromMap(manager.lowPriorityJobs);
- return job.isPresent() ? job.get() : null;
+ return job.isPresent() ? Pair.of(job.get(), JobPriority.LOW) : null;
}
protected Optional<Map.Entry<TableName, Set<String>>> fetchJobFromMap(Map<TableName, Set<String>> jobMap) {
@@ -112,12 +106,12 @@ public class StatisticsAutoCollector extends StatisticsCollector {
}
}
- protected void processOneJob(TableIf table, Set<String> columns) throws DdlException {
+ protected void processOneJob(TableIf table, Set<String> columns, JobPriority priority) throws DdlException {
appendPartitionColumns(table, columns);
if (columns.isEmpty()) {
return;
}
- AnalysisInfo analyzeJob = createAnalyzeJobForTbl(table, columns);
+ AnalysisInfo analyzeJob = createAnalyzeJobForTbl(table, columns, priority);
LOG.info("Analyze job : {}", analyzeJob.toString());
createSystemAnalysisJob(analyzeJob);
}
@@ -134,69 +128,6 @@ public class StatisticsAutoCollector extends StatisticsCollector {
}
}
- // TODO: Need refactor, hard to understand now.
- protected boolean needAnalyzeColumn(TableIf table, String column) {
- AnalysisManager manager = Env.getServingEnv().getAnalysisManager();
- TableStatsMeta tableStatsStatus = manager.findTableStatsStatus(table.getId());
- if (tableStatsStatus == null) {
- return true;
- }
- if (tableStatsStatus.userInjected) {
- return false;
- }
- ColStatsMeta columnStatsMeta = tableStatsStatus.findColumnStatsMeta(column);
- if (columnStatsMeta == null) {
- return true;
- }
- if (table instanceof OlapTable) {
- long currentUpdatedRows = tableStatsStatus.updatedRows.get();
- long lastAnalyzeUpdateRows = columnStatsMeta.updatedRows;
- if (lastAnalyzeUpdateRows == 0 && currentUpdatedRows > 0) {
- return true;
- }
- if (lastAnalyzeUpdateRows > currentUpdatedRows) {
- // Shouldn't happen. Just in case.
- return true;
- }
- OlapTable olapTable = (OlapTable) table;
- long currentRowCount = olapTable.getRowCount();
- long lastAnalyzeRowCount = columnStatsMeta.rowCount;
- if (tableStatsStatus.newPartitionLoaded.get() && olapTable.isPartitionColumn(column)) {
- return true;
- }
- if (lastAnalyzeRowCount == 0 && currentRowCount > 0) {
- return true;
- }
- if (currentUpdatedRows == lastAnalyzeUpdateRows) {
- return false;
- }
- double healthValue = ((double) (currentUpdatedRows - lastAnalyzeUpdateRows)
- / (double) currentUpdatedRows) * 100.0;
- LOG.info("Column " + column + " update rows health value is " + healthValue);
- if (healthValue < StatisticsUtil.getTableStatsHealthThreshold()) {
- return true;
- }
- if (currentRowCount == 0 && lastAnalyzeRowCount != 0) {
- return true;
- }
- if (currentRowCount == 0 && lastAnalyzeRowCount == 0) {
- return false;
- }
- healthValue = ((double) (currentRowCount - lastAnalyzeRowCount) / (double) currentRowCount) * 100.0;
- return healthValue < StatisticsUtil.getTableStatsHealthThreshold();
- } else {
- if (!(table instanceof HMSExternalTable)) {
- return false;
- }
- HMSExternalTable hmsTable = (HMSExternalTable) table;
- if (!hmsTable.getDlaType().equals(DLAType.HIVE)) {
- return false;
- }
- return System.currentTimeMillis()
- - tableStatsStatus.updatedTime > StatisticsUtil.getExternalTableAutoAnalyzeIntervalInMillis();
- }
- }
-
protected boolean supportAutoAnalyze(TableIf tableIf) {
if (tableIf == null) {
return false;
@@ -206,7 +137,7 @@ public class StatisticsAutoCollector extends StatisticsCollector {
&& ((HMSExternalTable) tableIf).getDlaType().equals(HMSExternalTable.DLAType.HIVE);
}
- protected AnalysisInfo createAnalyzeJobForTbl(TableIf table, Set<String> columns) {
+ protected AnalysisInfo createAnalyzeJobForTbl(TableIf table, Set<String> columns, JobPriority priority) {
AnalysisMethod analysisMethod = table.getDataSize(true) >= StatisticsUtil.getHugeTableLowerBoundSizeInBytes()
? AnalysisMethod.SAMPLE : AnalysisMethod.FULL;
AnalysisManager manager = Env.getServingEnv().getAnalysisManager();
@@ -236,6 +167,7 @@ public class StatisticsAutoCollector extends StatisticsCollector {
.setRowCount(rowCount)
.setUpdateRows(tableStatsStatus == null ? 0 : tableStatsStatus.updatedRows.get())
.setColToPartitions(colToPartitions)
+ .setPriority(priority)
.build();
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java
index 93d03a3fdb8..9e07c65e2fe 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java
@@ -86,6 +86,9 @@ public class StatisticsJobAppender extends MasterDaemon {
int size = columnQueue.size();
for (int i = 0; i < size; i++) {
HighPriorityColumn column = columnQueue.poll();
+ if (!StatisticsUtil.needAnalyzeColumn(column)) {
+ continue;
+ }
LOG.info("Process column " + column.tblId + "." + column.colName);
TableIf table = StatisticsUtil.findTable(column.catalogId, column.dbId, column.tblId);
TableName tableName = new TableName(table.getDatabase().getCatalog().getName(),
@@ -132,8 +135,9 @@ public class StatisticsJobAppender extends MasterDaemon {
if (!jobsMap.containsKey(tableName) && jobsMap.size() >= JOB_MAP_SIZE) {
return;
}
- Set<String> columns
- = t.getColumns().stream().filter(c -> !StatisticsUtil.isUnsupportedType(c.getType()))
+ Set<String> columns = t.getColumns().stream()
+ .filter(c -> !StatisticsUtil.isUnsupportedType(c.getType()))
+ .filter(c -> StatisticsUtil.needAnalyzeColumn(t, c.getName()))
.map(c -> c.getName()).collect(Collectors.toSet());
if (jobsMap.containsKey(tableName)) {
jobsMap.get(tableName).addAll(columns);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index 0a56a11d115..3ce8e7966af 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -59,6 +59,7 @@ import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.datasource.hive.HMSExternalCatalog;
import org.apache.doris.datasource.hive.HMSExternalTable;
+import org.apache.doris.datasource.hive.HMSExternalTable.DLAType;
import org.apache.doris.datasource.hive.HiveMetaStoreCache;
import org.apache.doris.datasource.hive.HivePartition;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
@@ -70,11 +71,15 @@ import org.apache.doris.qe.SessionVariable;
import org.apache.doris.qe.StmtExecutor;
import org.apache.doris.qe.VariableMgr;
import org.apache.doris.statistics.AnalysisInfo;
+import org.apache.doris.statistics.AnalysisManager;
+import org.apache.doris.statistics.ColStatsMeta;
import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.ColumnStatisticBuilder;
+import org.apache.doris.statistics.HighPriorityColumn;
import org.apache.doris.statistics.Histogram;
import org.apache.doris.statistics.ResultRow;
import org.apache.doris.statistics.StatisticConstants;
+import org.apache.doris.statistics.TableStatsMeta;
import org.apache.doris.system.Frontend;
import com.google.common.base.Preconditions;
@@ -905,7 +910,7 @@ public class StatisticsUtil {
} catch (Exception e) {
LOG.warn("Fail to get value of enable auto analyze internal catalog, return false by default", e);
}
- return false;
+ return true;
}
public static int getInsertMergeCount() {
@@ -1039,4 +1044,85 @@ public class StatisticsUtil {
return true;
}
+ // TODO: Need refactor, hard to understand now.
+ public static boolean needAnalyzeColumn(TableIf table, String column) {
+ AnalysisManager manager = Env.getServingEnv().getAnalysisManager();
+ TableStatsMeta tableStatsStatus = manager.findTableStatsStatus(table.getId());
+ if (tableStatsStatus == null) {
+ return true;
+ }
+ if (tableStatsStatus.userInjected) {
+ return false;
+ }
+ ColStatsMeta columnStatsMeta = tableStatsStatus.findColumnStatsMeta(column);
+ if (columnStatsMeta == null) {
+ return true;
+ }
+ if (table instanceof OlapTable) {
+ long currentUpdatedRows = tableStatsStatus.updatedRows.get();
+ long lastAnalyzeUpdateRows = columnStatsMeta.updatedRows;
+ if (lastAnalyzeUpdateRows == 0 && currentUpdatedRows > 0) {
+ return true;
+ }
+ if (lastAnalyzeUpdateRows > currentUpdatedRows) {
+ // Shouldn't happen. Just in case.
+ return true;
+ }
+ OlapTable olapTable = (OlapTable) table;
+ long currentRowCount = olapTable.getRowCount();
+ long lastAnalyzeRowCount = columnStatsMeta.rowCount;
+ if (tableStatsStatus.newPartitionLoaded.get() && olapTable.isPartitionColumn(column)) {
+ return true;
+ }
+ if (lastAnalyzeRowCount == 0 && currentRowCount > 0) {
+ return true;
+ }
+ if (currentUpdatedRows == lastAnalyzeUpdateRows) {
+ return false;
+ }
+ double healthValue = ((double) (currentUpdatedRows - lastAnalyzeUpdateRows)
+ / (double) currentUpdatedRows) * 100.0;
+ LOG.info("Column " + column + " update rows health value is " + healthValue);
+ if (healthValue < StatisticsUtil.getTableStatsHealthThreshold()) {
+ return true;
+ }
+ if (currentRowCount == 0 && lastAnalyzeRowCount != 0) {
+ return true;
+ }
+ if (currentRowCount == 0 && lastAnalyzeRowCount == 0) {
+ return false;
+ }
+ healthValue = ((double) (currentRowCount - lastAnalyzeRowCount) / (double) currentRowCount) * 100.0;
+ return healthValue < StatisticsUtil.getTableStatsHealthThreshold();
+ } else {
+ if (!(table instanceof HMSExternalTable)) {
+ return false;
+ }
+ HMSExternalTable hmsTable = (HMSExternalTable) table;
+ if (!hmsTable.getDlaType().equals(DLAType.HIVE)) {
+ return false;
+ }
+ return System.currentTimeMillis()
+ - tableStatsStatus.updatedTime > StatisticsUtil.getExternalTableAutoAnalyzeIntervalInMillis();
+ }
+ }
+
+ public static boolean needAnalyzeColumn(HighPriorityColumn column) {
+ if (column == null) {
+ return false;
+ }
+ TableIf table;
+ Column col;
+ try {
+ table = StatisticsUtil.findTable(column.catalogId, column.dbId, column.tblId);
+ col = table.getColumn(column.colName);
+ } catch (Exception e) {
+ LOG.warn("Failed to find table for column {}", column.colName, e);
+ return false;
+ }
+ return col != null
+ && !StatisticsUtil.isUnsupportedType(col.getType())
+ && StatisticsUtil.needAnalyzeColumn(table, column.colName);
+ }
+
}
diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift
index 2ddf52afd49..04f9fd9a322 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -1423,9 +1423,9 @@ struct TReportCommitTxnResultRequest {
}
struct TQueryColumn {
- 1: optional i64 catalogId
- 2: optional i64 dbId
- 3: optional i64 tblId
+ 1: optional string catalogId
+ 2: optional string dbId
+ 3: optional string tblId
4: optional string colName
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 05/09: Support show auto analyze pending jobs. (#31926)
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch high-priority-column
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 8631bcafdc3676a4c6ec36a6b81dee81b179bd86
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Thu Mar 7 15:02:02 2024 +0800
Support show auto analyze pending jobs. (#31926)
---
fe/fe-core/src/main/cup/sql_parser.cup | 4 +
.../doris/analysis/ShowAutoAnalyzeJobsStmt.java | 210 +++++++++++++++++++++
.../java/org/apache/doris/qe/ShowExecutor.java | 34 ++++
.../apache/doris/statistics/AnalysisManager.java | 40 +++-
.../doris/statistics/AutoAnalysisPendingJob.java | 50 +++++
.../org/apache/doris/statistics/JobPriority.java | 24 +++
.../doris/statistics/StatisticsAutoCollector.java | 16 +-
.../doris/statistics/StatisticsJobAppender.java | 25 ++-
8 files changed, 383 insertions(+), 20 deletions(-)
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup
index 22dd017b68a..149d03999f0 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -4361,6 +4361,10 @@ show_param ::=
{:
RESULT = new ShowAnalyzeStmt(tbl, parser.where, true);
:}
+ | KW_AUTO KW_JOBS opt_table_name:tbl opt_wild_where
+ {:
+ RESULT = new ShowAutoAnalyzeJobsStmt(tbl, parser.where);
+ :}
| KW_ANALYZE KW_TASK KW_STATUS INTEGER_LITERAL:jobId
{:
RESULT = new ShowAnalyzeTaskStatus(jobId);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAutoAnalyzeJobsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAutoAnalyzeJobsStmt.java
new file mode 100644
index 00000000000..560387fa5bc
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAutoAnalyzeJobsStmt.java
@@ -0,0 +1,210 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.UserException;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.statistics.JobPriority;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
+
+/**
+ * ShowAutoAnalyzeJobsStmt is used to show pending auto analysis jobs.
+ * syntax:
+ * SHOW AUTO ANALYZE JOBS
+ * [TABLE]
+ * [
+ * WHERE
+ * [PRIORITY = ["HIGH"|"MID"|"LOW"]]
+ * ]
+ */
+public class ShowAutoAnalyzeJobsStmt extends ShowStmt {
+ private static final String PRIORITY = "priority";
+ private static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>()
+ .add("catalog_name")
+ .add("db_name")
+ .add("tbl_name")
+ .add("col_list")
+ .add("priority")
+ .build();
+
+ private final TableName tableName;
+ private final Expr whereClause;
+
+ public ShowAutoAnalyzeJobsStmt(TableName tableName, Expr whereClause) {
+ this.tableName = tableName;
+ this.whereClause = whereClause;
+ }
+
+ // extract from predicate
+ private String jobPriority;
+
+ public String getPriority() {
+ Preconditions.checkArgument(isAnalyzed(),
+ "The stateValue must be obtained after the parsing is complete");
+ return jobPriority;
+ }
+
+ public Expr getWhereClause() {
+ Preconditions.checkArgument(isAnalyzed(),
+ "The whereClause must be obtained after the parsing is complete");
+ return whereClause;
+ }
+
+ @Override
+ public void analyze(Analyzer analyzer) throws UserException {
+ if (!ConnectContext.get().getSessionVariable().enableStats) {
+ throw new UserException("Analyze function is forbidden, you should add `enable_stats=true`"
+ + "in your FE conf file");
+ }
+ super.analyze(analyzer);
+ if (tableName != null) {
+ tableName.analyze(analyzer);
+ String catalogName = tableName.getCtl();
+ String dbName = tableName.getDb();
+ String tblName = tableName.getTbl();
+ checkShowAnalyzePriv(catalogName, dbName, tblName);
+ }
+
+ // analyze where clause if not null
+ if (whereClause != null) {
+ analyzeSubPredicate(whereClause);
+ }
+ }
+
+ @Override
+ public ShowResultSetMetaData getMetaData() {
+ ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
+ for (String title : TITLE_NAMES) {
+ builder.addColumn(new Column(title, ScalarType.createVarchar(128)));
+ }
+ return builder.build();
+ }
+
+ @Override
+ public RedirectStatus getRedirectStatus() {
+ return RedirectStatus.FORWARD_NO_SYNC;
+ }
+
+ private void checkShowAnalyzePriv(String catalogName, String dbName, String tblName) throws AnalysisException {
+ if (!Env.getCurrentEnv().getAccessManager()
+ .checkTblPriv(ConnectContext.get(), catalogName, dbName, tblName, PrivPredicate.SHOW)) {
+ ErrorReport.reportAnalysisException(
+ ErrorCode.ERR_TABLEACCESS_DENIED_ERROR,
+ "SHOW ANALYZE",
+ ConnectContext.get().getQualifiedUser(),
+ ConnectContext.get().getRemoteIP(),
+ dbName + ": " + tblName);
+ }
+ }
+
+ private void analyzeSubPredicate(Expr subExpr) throws AnalysisException {
+ if (subExpr == null) {
+ return;
+ }
+
+ boolean valid = true;
+
+ CHECK: {
+ if (subExpr instanceof BinaryPredicate) {
+ BinaryPredicate binaryPredicate = (BinaryPredicate) subExpr;
+ if (binaryPredicate.getOp() != BinaryPredicate.Operator.EQ) {
+ valid = false;
+ break CHECK;
+ }
+ } else {
+ valid = false;
+ break CHECK;
+ }
+
+ // left child
+ if (!(subExpr.getChild(0) instanceof SlotRef)) {
+ valid = false;
+ break CHECK;
+ }
+ String leftKey = ((SlotRef) subExpr.getChild(0)).getColumnName();
+ if (!PRIORITY.equalsIgnoreCase(leftKey)) {
+ valid = false;
+ break CHECK;
+ }
+
+ // right child
+ if (!(subExpr.getChild(1) instanceof StringLiteral)) {
+ valid = false;
+ break CHECK;
+ }
+
+ String value = subExpr.getChild(1).getStringValue();
+ if (Strings.isNullOrEmpty(value)) {
+ valid = false;
+ break CHECK;
+ }
+
+ jobPriority = value.toUpperCase();
+ try {
+ JobPriority.valueOf(jobPriority);
+ } catch (Exception e) {
+ valid = false;
+ }
+ }
+
+ if (!valid) {
+ throw new AnalysisException("Where clause should looks like: "
+ + "PRIORITY = \"HIGH|MID|LOW\"");
+ }
+ }
+
+ @Override
+ public String toSql() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("SHOW AUTO ANALYZE");
+
+ if (tableName != null) {
+ sb.append(" ");
+ sb.append(tableName.toSql());
+ }
+
+ if (whereClause != null) {
+ sb.append(" ");
+ sb.append("WHERE");
+ sb.append(" ");
+ sb.append(whereClause.toSql());
+ }
+
+ return sb.toString();
+ }
+
+ @Override
+ public String toString() {
+ return toSql();
+ }
+
+ public TableName getTableName() {
+ return tableName;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index 53ff62edb68..670ec85f6f7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -30,6 +30,7 @@ import org.apache.doris.analysis.ShowAlterStmt;
import org.apache.doris.analysis.ShowAnalyzeStmt;
import org.apache.doris.analysis.ShowAnalyzeTaskStatus;
import org.apache.doris.analysis.ShowAuthorStmt;
+import org.apache.doris.analysis.ShowAutoAnalyzeJobsStmt;
import org.apache.doris.analysis.ShowBackendsStmt;
import org.apache.doris.analysis.ShowBackupStmt;
import org.apache.doris.analysis.ShowBrokerStmt;
@@ -203,6 +204,7 @@ import org.apache.doris.mysql.privilege.PrivBitSet;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.mysql.privilege.Privilege;
import org.apache.doris.statistics.AnalysisInfo;
+import org.apache.doris.statistics.AutoAnalysisPendingJob;
import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.Histogram;
import org.apache.doris.statistics.ResultRow;
@@ -440,6 +442,8 @@ public class ShowExecutor {
handleShowCreateCatalog();
} else if (stmt instanceof ShowAnalyzeStmt) {
handleShowAnalyze();
+ } else if (stmt instanceof ShowAutoAnalyzeJobsStmt) {
+ handleShowAutoAnalyzePendingJobs();
} else if (stmt instanceof ShowTabletsBelongStmt) {
handleShowTabletsBelong();
} else if (stmt instanceof AdminCopyTabletStmt) {
@@ -2873,6 +2877,36 @@ public class ShowExecutor {
resultSet = new ShowResultSet(showStmt.getMetaData(), resultRows);
}
+ private void handleShowAutoAnalyzePendingJobs() {
+ ShowAutoAnalyzeJobsStmt showStmt = (ShowAutoAnalyzeJobsStmt) stmt;
+ List<AutoAnalysisPendingJob> jobs = Env.getCurrentEnv().getAnalysisManager().showAutoPendingJobs(showStmt);
+ List<List<String>> resultRows = Lists.newArrayList();
+ for (AutoAnalysisPendingJob job : jobs) {
+ try {
+ List<String> row = new ArrayList<>();
+ CatalogIf<? extends DatabaseIf<? extends TableIf>> c
+ = StatisticsUtil.findCatalog(job.catalogName);
+ row.add(c.getName());
+ Optional<? extends DatabaseIf<? extends TableIf>> databaseIf = c.getDb(job.dbName);
+ row.add(databaseIf.isPresent() ? databaseIf.get().getFullName() : "DB may get deleted");
+ if (databaseIf.isPresent()) {
+ Optional<? extends TableIf> table = databaseIf.get().getTable(job.tableName);
+ row.add(table.isPresent() ? table.get().getName() : "Table may get deleted");
+ } else {
+ row.add("DB may get deleted");
+ }
+ row.add(job.getColumnNames());
+ row.add(String.valueOf(job.priority));
+ resultRows.add(row);
+ } catch (Exception e) {
+ LOG.warn("Failed to get pending jobs for table {}.{}.{}, reason: {}",
+ job.catalogName, job.dbName, job.tableName, e.getMessage());
+ continue;
+ }
+ }
+ resultSet = new ShowResultSet(showStmt.getMetaData(), resultRows);
+ }
+
private void handleShowTabletsBelong() {
ShowTabletsBelongStmt showStmt = (ShowTabletsBelongStmt) stmt;
List<List<String>> rows = new ArrayList<>();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index e2bd85a0f43..c6713b3a7c1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -25,6 +25,7 @@ import org.apache.doris.analysis.DropAnalyzeJobStmt;
import org.apache.doris.analysis.DropStatsStmt;
import org.apache.doris.analysis.KillAnalysisJobStmt;
import org.apache.doris.analysis.ShowAnalyzeStmt;
+import org.apache.doris.analysis.ShowAutoAnalyzeJobsStmt;
import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DatabaseIf;
@@ -113,9 +114,9 @@ public class AnalysisManager implements Writable {
private static final int COLUMN_QUEUE_SIZE = 1000;
public final Queue<HighPriorityColumn> highPriorityColumns = new ArrayBlockingQueue<>(COLUMN_QUEUE_SIZE);
public final Queue<HighPriorityColumn> midPriorityColumns = new ArrayBlockingQueue<>(COLUMN_QUEUE_SIZE);
- public final Map<TableIf, Set<String>> highPriorityJobs = new LinkedHashMap<>();
- public final Map<TableIf, Set<String>> midPriorityJobs = new LinkedHashMap<>();
- public final Map<TableIf, Set<String>> lowPriorityJobs = new LinkedHashMap<>();
+ public final Map<TableName, Set<String>> highPriorityJobs = new LinkedHashMap<>();
+ public final Map<TableName, Set<String>> midPriorityJobs = new LinkedHashMap<>();
+ public final Map<TableName, Set<String>> lowPriorityJobs = new LinkedHashMap<>();
// Tracking running manually submitted async tasks, keep in mem only
protected final ConcurrentMap<Long, Map<Long, BaseAnalysisTask>> analysisJobIdToTaskMap = new ConcurrentHashMap<>();
@@ -598,6 +599,39 @@ public class AnalysisManager implements Writable {
}
}
+ public List<AutoAnalysisPendingJob> showAutoPendingJobs(ShowAutoAnalyzeJobsStmt stmt) {
+ TableName tblName = stmt.getTableName();
+ String priority = stmt.getPriority();
+ List<AutoAnalysisPendingJob> result = Lists.newArrayList();
+ if (priority == null || priority.isEmpty()) {
+ result.addAll(getPendingJobs(highPriorityJobs, JobPriority.HIGH, tblName));
+ result.addAll(getPendingJobs(midPriorityJobs, JobPriority.MID, tblName));
+ result.addAll(getPendingJobs(lowPriorityJobs, JobPriority.LOW, tblName));
+ } else if (priority.equals(JobPriority.HIGH.name())) {
+ result.addAll(getPendingJobs(highPriorityJobs, JobPriority.HIGH, tblName));
+ } else if (priority.equals(JobPriority.MID.name())) {
+ result.addAll(getPendingJobs(midPriorityJobs, JobPriority.MID, tblName));
+ } else if (priority.equals(JobPriority.LOW.name())) {
+ result.addAll(getPendingJobs(lowPriorityJobs, JobPriority.LOW, tblName));
+ }
+ return result;
+ }
+
+ protected List<AutoAnalysisPendingJob> getPendingJobs(Map<TableName, Set<String>> jobMap,
+ JobPriority priority, TableName tblName) {
+ List<AutoAnalysisPendingJob> result = Lists.newArrayList();
+ synchronized (jobMap) {
+ for (Entry<TableName, Set<String>> entry : jobMap.entrySet()) {
+ TableName table = entry.getKey();
+ if (tblName == null || tblName.equals(table)) {
+ result.add(new AutoAnalysisPendingJob(table.getCtl(),
+ table.getDb(), table.getTbl(), entry.getValue(), priority));
+ }
+ }
+ }
+ return result;
+ }
+
public List<AnalysisInfo> showAnalysisJob(ShowAnalyzeStmt stmt) {
return findShowAnalyzeResult(analysisJobInfoMap.values(), stmt);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AutoAnalysisPendingJob.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AutoAnalysisPendingJob.java
new file mode 100644
index 00000000000..ddd06d17c81
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AutoAnalysisPendingJob.java
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.statistics;
+
+import java.util.Set;
+import java.util.StringJoiner;
+
+public class AutoAnalysisPendingJob {
+
+ public final String catalogName;
+ public final String dbName;
+ public final String tableName;
+ public final Set<String> columnNames;
+ public final JobPriority priority;
+
+ public AutoAnalysisPendingJob(String catalogName, String dbName, String tableName,
+ Set<String> columnNames, JobPriority priority) {
+ this.catalogName = catalogName;
+ this.dbName = dbName;
+ this.tableName = tableName;
+ this.columnNames = columnNames;
+ this.priority = priority;
+ }
+
+ public String getColumnNames() {
+ if (columnNames == null) {
+ return "";
+ }
+ StringJoiner stringJoiner = new StringJoiner(",");
+ for (String colName : columnNames) {
+ stringJoiner.add(colName);
+ }
+ return stringJoiner.toString();
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/JobPriority.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/JobPriority.java
new file mode 100644
index 00000000000..2786b063563
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/JobPriority.java
@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.statistics;
+
+public enum JobPriority {
+ HIGH,
+ MID,
+ LOW;
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
index e0df94b5cb0..227074dbb5c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -17,6 +17,7 @@
package org.apache.doris.statistics;
+import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.TableIf;
@@ -57,13 +58,14 @@ public class StatisticsAutoCollector extends StatisticsCollector {
@Override
protected void collect() {
while (canCollect()) {
- Map.Entry<TableIf, Set<String>> job = getJob();
+ Map.Entry<TableName, Set<String>> job = getJob();
if (job == null) {
// No more job to process, break and sleep.
break;
}
try {
- TableIf table = job.getKey();
+ TableName tblName = job.getKey();
+ TableIf table = StatisticsUtil.findTable(tblName.getCtl(), tblName.getDb(), tblName.getTbl());
if (!supportAutoAnalyze(table)) {
continue;
}
@@ -78,7 +80,7 @@ public class StatisticsAutoCollector extends StatisticsCollector {
processOneJob(table, columns);
} catch (Exception e) {
LOG.warn("Failed to analyze table {} with columns [{}]",
- job.getKey().getName(), job.getValue().stream().collect(Collectors.joining(",")), e);
+ job.getKey().getTbl(), job.getValue().stream().collect(Collectors.joining(",")), e);
}
}
}
@@ -88,9 +90,9 @@ public class StatisticsAutoCollector extends StatisticsCollector {
&& StatisticsUtil.inAnalyzeTime(LocalTime.now(TimeUtils.getTimeZone().toZoneId()));
}
- protected Map.Entry<TableIf, Set<String>> getJob() {
+ protected Map.Entry<TableName, Set<String>> getJob() {
AnalysisManager manager = Env.getServingEnv().getAnalysisManager();
- Optional<Map.Entry<TableIf, Set<String>>> job = fetchJobFromMap(manager.highPriorityJobs);
+ Optional<Map.Entry<TableName, Set<String>>> job = fetchJobFromMap(manager.highPriorityJobs);
if (job.isPresent()) {
return job.get();
}
@@ -102,9 +104,9 @@ public class StatisticsAutoCollector extends StatisticsCollector {
return job.isPresent() ? job.get() : null;
}
- protected Optional<Map.Entry<TableIf, Set<String>>> fetchJobFromMap(Map<TableIf, Set<String>> jobMap) {
+ protected Optional<Map.Entry<TableName, Set<String>>> fetchJobFromMap(Map<TableName, Set<String>> jobMap) {
synchronized (jobMap) {
- Optional<Map.Entry<TableIf, Set<String>>> first = jobMap.entrySet().stream().findFirst();
+ Optional<Map.Entry<TableName, Set<String>>> first = jobMap.entrySet().stream().findFirst();
first.ifPresent(entry -> jobMap.remove(entry.getKey()));
return first;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java
index 71bb71d3cda..93d03a3fdb8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java
@@ -17,6 +17,7 @@
package org.apache.doris.statistics;
+import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
@@ -81,31 +82,33 @@ public class StatisticsJobAppender extends MasterDaemon {
}
}
- protected void appendColumnsToJobs(Queue<HighPriorityColumn> columnQueue, Map<TableIf, Set<String>> jobsMap) {
+ protected void appendColumnsToJobs(Queue<HighPriorityColumn> columnQueue, Map<TableName, Set<String>> jobsMap) {
int size = columnQueue.size();
for (int i = 0; i < size; i++) {
HighPriorityColumn column = columnQueue.poll();
LOG.info("Process column " + column.tblId + "." + column.colName);
TableIf table = StatisticsUtil.findTable(column.catalogId, column.dbId, column.tblId);
+ TableName tableName = new TableName(table.getDatabase().getCatalog().getName(),
+ table.getDatabase().getFullName(), table.getName());
synchronized (jobsMap) {
// If job map reach the upper limit, stop putting new jobs.
- if (!jobsMap.containsKey(table) && jobsMap.size() >= JOB_MAP_SIZE) {
+ if (!jobsMap.containsKey(tableName) && jobsMap.size() >= JOB_MAP_SIZE) {
LOG.info("Job map full.");
break;
}
- if (jobsMap.containsKey(table)) {
- jobsMap.get(table).add(column.colName);
+ if (jobsMap.containsKey(tableName)) {
+ jobsMap.get(tableName).add(column.colName);
} else {
HashSet<String> columns = new HashSet<>();
columns.add(column.colName);
- jobsMap.put(table, columns);
+ jobsMap.put(tableName, columns);
}
LOG.info("Column " + column.tblId + "." + column.colName + " added");
}
}
}
- protected void appendToLowQueue(Map<TableIf, Set<String>> jobsMap) {
+ protected void appendToLowQueue(Map<TableName, Set<String>> jobsMap) {
InternalCatalog catalog = Env.getCurrentInternalCatalog();
List<Long> sortedDbs = catalog.getDbIds().stream().sorted().collect(Collectors.toList());
int batchSize = 100;
@@ -122,18 +125,20 @@ public class StatisticsJobAppender extends MasterDaemon {
if (!(t instanceof OlapTable) || t.getId() <= currentTableId) {
continue;
}
+ TableName tableName = new TableName(t.getDatabase().getCatalog().getName(),
+ t.getDatabase().getFullName(), t.getName());
synchronized (jobsMap) {
// If job map reach the upper limit, stop adding new jobs.
- if (!jobsMap.containsKey(t) && jobsMap.size() >= JOB_MAP_SIZE) {
+ if (!jobsMap.containsKey(tableName) && jobsMap.size() >= JOB_MAP_SIZE) {
return;
}
Set<String> columns
= t.getColumns().stream().filter(c -> !StatisticsUtil.isUnsupportedType(c.getType()))
.map(c -> c.getName()).collect(Collectors.toSet());
- if (jobsMap.containsKey(t)) {
- jobsMap.get(t).addAll(columns);
+ if (jobsMap.containsKey(tableName)) {
+ jobsMap.get(tableName).addAll(columns);
} else {
- jobsMap.put(t, columns);
+ jobsMap.put(tableName, columns);
}
}
currentTableId = t.getId();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 01/09: Collect high priority columns. (#31235)
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch high-priority-column
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1bc42cd821a6ec74c9c0f3a5db41a6fbd63f07f5
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Wed Feb 28 11:54:40 2024 +0800
Collect high priority columns. (#31235)
---
.../doris/nereids/jobs/executor/Rewriter.java | 4 +-
.../org/apache/doris/nereids/rules/RuleType.java | 1 +
.../expression/HighPriorityColumnCollector.java | 202 +++++++++++++++++++++
.../apache/doris/statistics/AnalysisManager.java | 44 +++++
.../doris/statistics/HighPriorityColumn.java | 55 ++++++
5 files changed, 305 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index 6dee5ef18f8..9977a36879c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -32,6 +32,7 @@ import org.apache.doris.nereids.rules.expression.CheckLegalityAfterRewrite;
import org.apache.doris.nereids.rules.expression.ExpressionNormalization;
import org.apache.doris.nereids.rules.expression.ExpressionOptimization;
import org.apache.doris.nereids.rules.expression.ExpressionRewrite;
+import org.apache.doris.nereids.rules.expression.HighPriorityColumnCollector;
import org.apache.doris.nereids.rules.rewrite.AddDefaultLimit;
import org.apache.doris.nereids.rules.rewrite.AdjustConjunctsReturnType;
import org.apache.doris.nereids.rules.rewrite.AdjustNullable;
@@ -408,7 +409,8 @@ public class Rewriter extends AbstractBatchJobExecutor {
new CollectFilterAboveConsumer(),
new CollectProjectAboveConsumer()
)
- )
+ ),
+ topic("Collect used column", custom(RuleType.COLLECT_COLUMNS, HighPriorityColumnCollector::new))
);
private static final List<RewriteJob> WHOLE_TREE_REWRITE_JOBS
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 672f78163ce..16977be2c9b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -303,6 +303,7 @@ public enum RuleType {
LEADING_JOIN(RuleTypeClass.REWRITE),
REWRITE_SENTINEL(RuleTypeClass.REWRITE),
+ COLLECT_COLUMNS(RuleTypeClass.REWRITE),
// topn opts
DEFER_MATERIALIZE_TOP_N_RESULT(RuleTypeClass.REWRITE),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/HighPriorityColumnCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/HighPriorityColumnCollector.java
new file mode 100644
index 00000000000..ed67ad97005
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/HighPriorityColumnCollector.java
@@ -0,0 +1,202 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.expression;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.nereids.jobs.JobContext;
+import org.apache.doris.nereids.rules.expression.HighPriorityColumnCollector.CollectorContext;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalHaving;
+import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter;
+import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.statistics.AnalysisManager;
+import org.apache.doris.statistics.util.StatisticsUtil;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Used to collect High priority column.
+ */
+public class HighPriorityColumnCollector extends DefaultPlanRewriter<CollectorContext> implements CustomRewriter {
+
+ @Override
+ public Plan rewriteRoot(Plan plan, JobContext jobContext) {
+ ConnectContext connectContext = ConnectContext.get();
+ if (connectContext != null && connectContext.getSessionVariable().internalSession) {
+ return plan;
+ }
+ CollectorContext context = new CollectorContext();
+ plan.accept(this, context);
+ if (StatisticsUtil.enableAutoAnalyze()) {
+ context.queried.removeAll(context.usedInPredicate);
+ AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
+ analysisManager.updateColumnUsedInPredicate(context.usedInPredicate);
+ analysisManager.updateQueriedColumn(context.queried);
+ }
+ return plan;
+ }
+
+ /**
+ * Context.
+ */
+ public static class CollectorContext {
+ public Map<Slot/*project output column*/, NamedExpression/*Actual project expr*/> projects = new HashMap<>();
+
+ public Set<Slot> usedInPredicate = new HashSet<>();
+
+ public Set<Slot> queried = new HashSet<>();
+ }
+
+ @Override
+ public Plan visitLogicalProject(LogicalProject<? extends Plan> project, CollectorContext context) {
+ project.child().accept(this, context);
+ List<NamedExpression> projects = project.getOutputs();
+ List<Slot> slots = project.computeOutput();
+ for (int i = 0; i < slots.size(); i++) {
+ context.projects.put(slots.get(i), projects.get(i));
+ }
+ if (project.child() instanceof LogicalCatalogRelation
+ || project.child() instanceof LogicalFilter
+ && ((LogicalFilter) project.child()).child() instanceof LogicalCatalogRelation) {
+ Set<Slot> allUsed = project.getExpressions()
+ .stream().flatMap(e -> e.<Set<SlotReference>>collect(n -> n instanceof SlotReference).stream())
+ .collect(Collectors.toSet());
+ LogicalCatalogRelation scan = project.child() instanceof LogicalCatalogRelation
+ ? (LogicalCatalogRelation) project.child()
+ : (LogicalCatalogRelation) project.child().child(0);
+ List<Slot> outputOfScan = scan.getOutput();
+ for (Slot slot : outputOfScan) {
+ if (!allUsed.contains(slot)) {
+ context.queried.remove(slot);
+ }
+ }
+ }
+ return project;
+ }
+
+ @Override
+ public Plan visitLogicalJoin(LogicalJoin<? extends Plan, ? extends Plan> join, CollectorContext context) {
+ join.child(0).accept(this, context);
+ join.child(1).accept(this, context);
+ context.usedInPredicate.addAll(
+ (join.isMarkJoin() ? join.getLeftConditionSlot() : join.getConditionSlot())
+ .stream().flatMap(s -> backtrace(s, context).stream())
+ .collect(Collectors.toSet())
+ );
+ return join;
+ }
+
+ @Override
+ public Plan visitLogicalAggregate(LogicalAggregate<? extends Plan> aggregate, CollectorContext context) {
+ aggregate.child(0).accept(this, context);
+ context.usedInPredicate.addAll(aggregate.getGroupByExpressions()
+ .stream()
+ .flatMap(e -> e.<Set<SlotReference>>collect(n -> n instanceof SlotReference).stream())
+ .flatMap(s -> backtrace(s, context).stream())
+ .collect(Collectors.toSet()));
+ return aggregate;
+ }
+
+ @Override
+ public Plan visitLogicalHaving(LogicalHaving<? extends Plan> having, CollectorContext context) {
+ having.child(0).accept(this, context);
+ context.usedInPredicate.addAll(
+ having.getExpressions().stream()
+ .flatMap(e -> e.<Set<SlotReference>>collect(n -> n instanceof SlotReference).stream())
+ .flatMap(s -> backtrace(s, context).stream())
+ .collect(Collectors.toSet()));
+ return having;
+ }
+
+ @Override
+ public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, CollectorContext context) {
+ List<Slot> slots = olapScan.getOutput();
+ context.queried.addAll(slots);
+ return olapScan;
+ }
+
+ @Override
+ public Plan visitLogicalFileScan(LogicalFileScan fileScan, CollectorContext context) {
+ List<Slot> slots = fileScan.getOutput();
+ context.queried.addAll(slots);
+ return fileScan;
+ }
+
+ @Override
+ public Plan visitLogicalFilter(LogicalFilter<? extends Plan> filter, CollectorContext context) {
+ filter.child(0).accept(this, context);
+ context.usedInPredicate.addAll(filter
+ .getExpressions()
+ .stream()
+ .flatMap(e -> e.<Set<SlotReference>>collect(n -> n instanceof SlotReference).stream())
+ .flatMap(s -> backtrace(s, context).stream())
+ .collect(Collectors.toSet()));
+ return filter;
+ }
+
+ private Set<Slot> backtrace(Slot slot, CollectorContext context) {
+ return backtrace(slot, new HashSet<>(), context);
+ }
+
+ private Set<Slot> backtrace(Slot slot, Set<Slot> path, CollectorContext context) {
+ if (path.contains(slot)) {
+ return Collections.emptySet();
+ }
+ path.add(slot);
+ if (slot instanceof SlotReference) {
+ SlotReference slotReference = (SlotReference) slot;
+ Optional<Column> col = slotReference.getColumn();
+ Optional<TableIf> table = slotReference.getTable();
+ if (col.isPresent() && table.isPresent()) {
+ return Collections.singleton(slot);
+ }
+ }
+ NamedExpression namedExpression = context.projects.get(slot);
+ if (namedExpression == null) {
+ return Collections.emptySet();
+ }
+ Set<SlotReference> slotReferences
+ = namedExpression.<Set<SlotReference>>collect(n -> n instanceof SlotReference);
+ Set<Slot> refCol = new HashSet<>();
+ for (SlotReference slotReference : slotReferences) {
+ refCol.addAll(backtrace(slotReference, path, context));
+ }
+ return refCol;
+ }
+
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index f90ce83a088..cf3764719ba 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -47,6 +47,8 @@ import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.persist.AnalyzeDeletionLog;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.qe.ConnectContext;
@@ -89,9 +91,11 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.NavigableMap;
import java.util.Optional;
+import java.util.Queue;
import java.util.Set;
import java.util.StringJoiner;
import java.util.TreeMap;
+import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
@@ -104,6 +108,16 @@ public class AnalysisManager implements Writable {
private static final Logger LOG = LogManager.getLogger(AnalysisManager.class);
+ /**
+ * Mem only.
+ */
+ public final Queue<HighPriorityColumn> predicateColumns = new ArrayBlockingQueue<>(100);
+
+ /**
+ * Mem only.
+ */
+ public final Queue<HighPriorityColumn> queryColumns = new ArrayBlockingQueue<>(100);
+
// Tracking running manually submitted async tasks, keep in mem only
protected final ConcurrentMap<Long, Map<Long, BaseAnalysisTask>> analysisJobIdToTaskMap = new ConcurrentHashMap<>();
@@ -1141,4 +1155,34 @@ public class AnalysisManager implements Writable {
}
return false;
}
+
+
+ public void updateColumnUsedInPredicate(Set<Slot> slotReferences) {
+ updateColumn(slotReferences, predicateColumns);
+ }
+
+ public void updateQueriedColumn(Collection<Slot> slotReferences) {
+ updateColumn(slotReferences, queryColumns);
+ }
+
+ protected void updateColumn(Collection<Slot> slotReferences, Queue<HighPriorityColumn> queue) {
+ for (Slot s : slotReferences) {
+ if (!(s instanceof SlotReference)) {
+ return;
+ }
+ Optional<Column> optionalColumn = ((SlotReference) s).getColumn();
+ Optional<TableIf> optionalTable = ((SlotReference) s).getTable();
+ if (optionalColumn.isPresent() && optionalTable.isPresent()) {
+ TableIf table = optionalTable.get();
+ DatabaseIf database = table.getDatabase();
+ if (database != null) {
+ CatalogIf catalog = database.getCatalog();
+ if (catalog != null) {
+ queue.offer(new HighPriorityColumn(catalog.getId(), database.getId(),
+ table.getId(), optionalColumn.get().getName()));
+ }
+ }
+ }
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HighPriorityColumn.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HighPriorityColumn.java
new file mode 100644
index 00000000000..c4bc20c399a
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HighPriorityColumn.java
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.statistics;
+
+import java.util.Objects;
+
+public class HighPriorityColumn {
+
+ public final long catalogId;
+ public final long dbId;
+ public final long tblId;
+ public final String colName;
+
+ public HighPriorityColumn(long catalogId, long dbId, long tblId, String colName) {
+ this.catalogId = catalogId;
+ this.dbId = dbId;
+ this.tblId = tblId;
+ this.colName = colName;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(catalogId, dbId, tblId, colName);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (!(other instanceof HighPriorityColumn)) {
+ return false;
+ }
+ HighPriorityColumn otherCriticalColumn = (HighPriorityColumn) other;
+ return this.catalogId == otherCriticalColumn.catalogId
+ && this.dbId == otherCriticalColumn.dbId
+ && this.tblId == otherCriticalColumn.tblId
+ && this.colName.equals(otherCriticalColumn.colName);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 02/09: High priority queue and map. (#31509)
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch high-priority-column
in repository https://gitbox.apache.org/repos/asf/doris.git
commit a4cba9f4b7ec39a7c14fd735c6c9c398999ef4aa
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Wed Feb 28 12:01:55 2024 +0800
High priority queue and map. (#31509)
---
.../main/java/org/apache/doris/catalog/Env.java | 20 +-
.../apache/doris/statistics/AnalysisManager.java | 33 +-
.../doris/statistics/StatisticsAutoCollector.java | 202 ++------
.../doris/statistics/StatisticsCollector.java | 4 -
.../doris/statistics/StatisticsJobAppender.java | 135 +++++
.../statistics/StatisticsAutoCollectorTest.java | 560 ---------------------
6 files changed, 213 insertions(+), 741 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 1a93efa0c5a..ba31670cbc2 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -243,6 +243,7 @@ import org.apache.doris.statistics.AnalysisManager;
import org.apache.doris.statistics.StatisticsAutoCollector;
import org.apache.doris.statistics.StatisticsCache;
import org.apache.doris.statistics.StatisticsCleaner;
+import org.apache.doris.statistics.StatisticsJobAppender;
import org.apache.doris.statistics.query.QueryStats;
import org.apache.doris.system.Backend;
import org.apache.doris.system.Frontend;
@@ -519,6 +520,8 @@ public class Env {
private StatisticsAutoCollector statisticsAutoCollector;
+ private StatisticsJobAppender statisticsJobAppender;
+
private HiveTransactionMgr hiveTransactionMgr;
private TopicPublisherThread topicPublisherThread;
@@ -748,6 +751,7 @@ public class Env {
this.analysisManager = new AnalysisManager();
this.statisticsCleaner = new StatisticsCleaner();
this.statisticsAutoCollector = new StatisticsAutoCollector();
+ this.statisticsJobAppender = new StatisticsJobAppender();
this.globalFunctionMgr = new GlobalFunctionMgr();
this.workloadGroupMgr = new WorkloadGroupMgr();
this.workloadSchedPolicyMgr = new WorkloadSchedPolicyMgr();
@@ -1028,13 +1032,6 @@ public class Env {
// If not using bdb, we need to notify the FE type transfer manually.
notifyNewFETypeTransfer(FrontendNodeType.MASTER);
}
- if (statisticsCleaner != null) {
- statisticsCleaner.start();
- }
- if (statisticsAutoCollector != null) {
- statisticsAutoCollector.start();
- }
-
queryCancelWorker.start();
TopicPublisher wgPublisher = new WorkloadGroupPublisher(this);
@@ -1672,6 +1669,11 @@ public class Env {
binlogGcer.start();
columnIdFlusher.start();
insertOverwriteManager.start();
+
+ // auto analyze related threads.
+ statisticsCleaner.start();
+ statisticsAutoCollector.start();
+ statisticsJobAppender.start();
}
// start threads that should running on all FE
@@ -6027,6 +6029,10 @@ public class Env {
return statisticsAutoCollector;
}
+ public StatisticsJobAppender getStatisticsJobAppender() {
+ return statisticsJobAppender;
+ }
+
public void alterMTMVRefreshInfo(AlterMTMVRefreshInfo info) {
AlterMTMV alter = new AlterMTMV(info.getMvName(), info.getRefreshInfo(), MTMVAlterOpType.ALTER_REFRESH_INFO);
this.alter.processAlterMTMV(alter, false);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index cf3764719ba..ab883c79f8f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -85,6 +85,7 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -108,15 +109,12 @@ public class AnalysisManager implements Writable {
private static final Logger LOG = LogManager.getLogger(AnalysisManager.class);
- /**
- * Mem only.
- */
- public final Queue<HighPriorityColumn> predicateColumns = new ArrayBlockingQueue<>(100);
-
- /**
- * Mem only.
- */
- public final Queue<HighPriorityColumn> queryColumns = new ArrayBlockingQueue<>(100);
+ private static final int COLUMN_QUEUE_SIZE = 1000;
+ public final Queue<HighPriorityColumn> highPriorityColumns = new ArrayBlockingQueue<>(COLUMN_QUEUE_SIZE);
+ public final Queue<HighPriorityColumn> midPriorityColumns = new ArrayBlockingQueue<>(COLUMN_QUEUE_SIZE);
+ public final Map<TableIf, Set<String>> highPriorityJobs = new LinkedHashMap<>();
+ public final Map<TableIf, Set<String>> midPriorityJobs = new LinkedHashMap<>();
+ public final Map<TableIf, Set<String>> lowPriorityJobs = new LinkedHashMap<>();
// Tracking running manually submitted async tasks, keep in mem only
protected final ConcurrentMap<Long, Map<Long, BaseAnalysisTask>> analysisJobIdToTaskMap = new ConcurrentHashMap<>();
@@ -170,11 +168,6 @@ public class AnalysisManager implements Writable {
public void createAnalysisJobs(AnalyzeDBStmt analyzeDBStmt, boolean proxy) throws DdlException, AnalysisException {
DatabaseIf<TableIf> db = analyzeDBStmt.getDb();
- // Using auto analyzer if user specifies.
- if (analyzeDBStmt.getAnalyzeProperties().getProperties().containsKey("use.auto.analyzer")) {
- Env.getCurrentEnv().getStatisticsAutoCollector().analyzeDb(db);
- return;
- }
List<AnalysisInfo> analysisInfos = buildAnalysisInfosForDB(db, analyzeDBStmt.getAnalyzeProperties());
if (!analyzeDBStmt.isSync()) {
sendJobId(analysisInfos, proxy);
@@ -222,6 +215,11 @@ public class AnalysisManager implements Writable {
// Each analyze stmt corresponding to an analysis job.
public void createAnalysisJob(AnalyzeTblStmt stmt, boolean proxy) throws DdlException {
+ // Using auto analyzer if user specifies.
+ if (stmt.getAnalyzeProperties().getProperties().containsKey("use.auto.analyzer")) {
+ Env.getCurrentEnv().getStatisticsAutoCollector().processOneJob(stmt.getTable(), stmt.getColumnNames());
+ return;
+ }
AnalysisInfo jobInfo = buildAndAssignJob(stmt);
if (jobInfo == null) {
return;
@@ -1158,11 +1156,11 @@ public class AnalysisManager implements Writable {
public void updateColumnUsedInPredicate(Set<Slot> slotReferences) {
- updateColumn(slotReferences, predicateColumns);
+ updateColumn(slotReferences, highPriorityColumns);
}
public void updateQueriedColumn(Collection<Slot> slotReferences) {
- updateColumn(slotReferences, queryColumns);
+ updateColumn(slotReferences, midPriorityColumns);
}
protected void updateColumn(Collection<Slot> slotReferences, Queue<HighPriorityColumn> queue) {
@@ -1172,7 +1170,8 @@ public class AnalysisManager implements Writable {
}
Optional<Column> optionalColumn = ((SlotReference) s).getColumn();
Optional<TableIf> optionalTable = ((SlotReference) s).getTable();
- if (optionalColumn.isPresent() && optionalTable.isPresent()) {
+ if (optionalColumn.isPresent() && optionalTable.isPresent()
+ && !StatisticsUtil.isUnsupportedType(optionalColumn.get().getType())) {
TableIf table = optionalTable.get();
DatabaseIf database = table.getDatabase();
if (database != null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
index dbb7046467a..19b9c69db08 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -17,31 +17,24 @@
package org.apache.doris.statistics;
-import org.apache.doris.catalog.Column;
-import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
-import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.util.TimeUtils;
-import org.apache.doris.datasource.CatalogIf;
-import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod;
import org.apache.doris.statistics.AnalysisInfo.JobType;
import org.apache.doris.statistics.AnalysisInfo.ScheduleType;
import org.apache.doris.statistics.util.StatisticsUtil;
-import org.apache.hudi.common.util.VisibleForTesting;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.time.LocalTime;
import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -59,8 +52,23 @@ public class StatisticsAutoCollector extends StatisticsCollector {
@Override
protected void collect() {
- if (canCollect()) {
- analyzeAll();
+ while (canCollect()) {
+ Map.Entry<TableIf, Set<String>> job = getJob();
+ if (job == null) {
+ // No more job to process, break and sleep.
+ break;
+ }
+ try {
+ TableIf table = job.getKey();
+ Set<String> columns = job.getValue()
+ .stream()
+ .filter(c -> needAnalyzeColumn(table, c))
+ .collect(Collectors.toSet());
+ processOneJob(table, columns);
+ } catch (Exception e) {
+ LOG.warn("Failed to analyze table {} with columns [{}]",
+ job.getKey().getName(), job.getValue().stream().collect(Collectors.joining(",")), e);
+ }
}
}
@@ -69,133 +77,56 @@ public class StatisticsAutoCollector extends StatisticsCollector {
&& StatisticsUtil.inAnalyzeTime(LocalTime.now(TimeUtils.getTimeZone().toZoneId()));
}
- protected void analyzeAll() {
- List<CatalogIf> catalogs = getCatalogsInOrder();
- for (CatalogIf ctl : catalogs) {
- if (!canCollect()) {
- analysisTaskExecutor.clear();
- break;
- }
- if (!ctl.enableAutoAnalyze()) {
- continue;
- }
- List<DatabaseIf> dbs = getDatabasesInOrder(ctl);
- for (DatabaseIf<TableIf> databaseIf : dbs) {
- if (!canCollect()) {
- analysisTaskExecutor.clear();
- break;
- }
- if (StatisticConstants.SYSTEM_DBS.contains(databaseIf.getFullName())) {
- continue;
- }
- try {
- analyzeDb(databaseIf);
- } catch (Throwable t) {
- LOG.warn("Failed to analyze database {}.{}", ctl.getName(), databaseIf.getFullName(), t);
- continue;
- }
- }
+ protected Map.Entry<TableIf, Set<String>> getJob() {
+ AnalysisManager manager = Env.getServingEnv().getAnalysisManager();
+ Optional<Map.Entry<TableIf, Set<String>>> job = fetchJobFromMap(manager.highPriorityJobs);
+ if (job.isPresent()) {
+ return job.get();
}
+ job = fetchJobFromMap(manager.midPriorityJobs);
+ if (job.isPresent()) {
+ return job.get();
+ }
+ job = fetchJobFromMap(manager.lowPriorityJobs);
+ return job.isPresent() ? job.get() : null;
}
- public List<CatalogIf> getCatalogsInOrder() {
- return Env.getCurrentEnv().getCatalogMgr().getCopyOfCatalog().stream()
- .sorted((c1, c2) -> (int) (c1.getId() - c2.getId())).collect(Collectors.toList());
- }
-
- public List<DatabaseIf<? extends TableIf>> getDatabasesInOrder(CatalogIf<DatabaseIf> catalog) {
- return catalog.getAllDbs().stream()
- .sorted((d1, d2) -> (int) (d1.getId() - d2.getId())).collect(Collectors.toList());
- }
-
- public List<TableIf> getTablesInOrder(DatabaseIf<? extends TableIf> db) {
- return db.getTables().stream()
- .sorted((t1, t2) -> (int) (t1.getId() - t2.getId())).collect(Collectors.toList());
- }
-
- public void analyzeDb(DatabaseIf<TableIf> databaseIf) throws DdlException {
- List<AnalysisInfo> analysisInfos = constructAnalysisInfo(databaseIf);
- for (AnalysisInfo analysisInfo : analysisInfos) {
- try {
- if (!canCollect()) {
- analysisTaskExecutor.clear();
- break;
- }
- analysisInfo = getReAnalyzeRequiredPart(analysisInfo);
- if (analysisInfo == null) {
- continue;
- }
- createSystemAnalysisJob(analysisInfo);
- } catch (Throwable t) {
- analysisInfo.message = t.getMessage();
- LOG.warn("Failed to auto analyze table {}.{}, reason {}",
- databaseIf.getFullName(), analysisInfo.tblId, analysisInfo.message, t);
- continue;
- }
+ protected Optional<Map.Entry<TableIf, Set<String>>> fetchJobFromMap(Map<TableIf, Set<String>> jobMap) {
+ synchronized (jobMap) {
+ Optional<Map.Entry<TableIf, Set<String>>> first = jobMap.entrySet().stream().findFirst();
+ first.ifPresent(entry -> jobMap.remove(entry.getKey()));
+ return first;
}
}
- protected List<AnalysisInfo> constructAnalysisInfo(DatabaseIf<? extends TableIf> db) {
- List<AnalysisInfo> analysisInfos = new ArrayList<>();
- for (TableIf table : getTablesInOrder(db)) {
- try {
- if (skip(table)) {
- continue;
- }
- createAnalyzeJobForTbl(db, analysisInfos, table);
- } catch (Throwable t) {
- LOG.warn("Failed to analyze table {}.{}.{}",
- db.getCatalog().getName(), db.getFullName(), table.getName(), t);
- continue;
- }
+ protected void processOneJob(TableIf table, Set<String> columns) throws DdlException {
+ Set<String> collect = columns.stream().filter(c -> needAnalyzeColumn(table, c)).collect(Collectors.toSet());
+ if (collect.isEmpty()) {
+ return;
}
- return analysisInfos;
+ AnalysisInfo analyzeJob = createAnalyzeJobForTbl(table, columns);
+ createSystemAnalysisJob(analyzeJob);
}
- // return true if skip auto analyze this time.
- protected boolean skip(TableIf table) {
- if (!(table instanceof OlapTable || table instanceof HMSExternalTable)) {
- return true;
- }
- // For now, only support Hive HMS table auto collection.
- if (table instanceof HMSExternalTable
- && !((HMSExternalTable) table).getDlaType().equals(HMSExternalTable.DLAType.HIVE)) {
- return true;
- }
- if (table.getDataSize(true) < StatisticsUtil.getHugeTableLowerBoundSizeInBytes() * 5) {
- return false;
- }
- TableStatsMeta tableStats = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(table.getId());
- // means it's never got analyzed or new partition loaded data.
- if (tableStats == null || tableStats.newPartitionLoaded.get()) {
- return false;
- }
- if (tableStats.userInjected) {
- return true;
- }
- return System.currentTimeMillis()
- - tableStats.updatedTime < StatisticsUtil.getHugeTableAutoAnalyzeIntervalInMillis();
+ protected boolean needAnalyzeColumn(TableIf table, String column) {
+ //TODO: Calculate column health value.
+ return true;
}
- protected void createAnalyzeJobForTbl(DatabaseIf<? extends TableIf> db,
- List<AnalysisInfo> analysisInfos, TableIf table) {
+ protected AnalysisInfo createAnalyzeJobForTbl(TableIf table, Set<String> columns) {
AnalysisMethod analysisMethod = table.getDataSize(true) >= StatisticsUtil.getHugeTableLowerBoundSizeInBytes()
? AnalysisMethod.SAMPLE : AnalysisMethod.FULL;
- AnalysisInfo jobInfo = new AnalysisInfoBuilder()
+ return new AnalysisInfoBuilder()
.setJobId(Env.getCurrentEnv().getNextId())
- .setCatalogId(db.getCatalog().getId())
- .setDBId(db.getId())
+ .setCatalogId(table.getDatabase().getCatalog().getId())
+ .setDBId(table.getDatabase().getId())
.setTblId(table.getId())
- .setColName(
- table.getSchemaAllIndexes(false).stream()
- .filter(c -> !StatisticsUtil.isUnsupportedType(c.getType()))
- .map(Column::getName).collect(Collectors.joining(","))
- )
+ .setColName(columns.stream().collect(Collectors.joining(",")))
.setAnalysisType(AnalysisInfo.AnalysisType.FUNDAMENTALS)
.setAnalysisMode(AnalysisInfo.AnalysisMode.INCREMENTAL)
.setAnalysisMethod(analysisMethod)
.setSampleRows(analysisMethod.equals(AnalysisMethod.SAMPLE)
- ? StatisticsUtil.getHugeTableSampleRows() : -1)
+ ? StatisticsUtil.getHugeTableSampleRows() : -1)
.setScheduleType(ScheduleType.AUTOMATIC)
.setState(AnalysisState.PENDING)
.setTaskIds(new ArrayList<>())
@@ -204,40 +135,5 @@ public class StatisticsAutoCollector extends StatisticsCollector {
.setTblUpdateTime(table.getUpdateTime())
.setEmptyJob(table instanceof OlapTable && table.getRowCount() == 0)
.build();
- analysisInfos.add(jobInfo);
- }
-
- @VisibleForTesting
- protected AnalysisInfo getReAnalyzeRequiredPart(AnalysisInfo jobInfo) {
- TableIf table = StatisticsUtil.findTable(jobInfo.catalogId, jobInfo.dbId, jobInfo.tblId);
- // Skip tables that are too wide.
- if (table.getBaseSchema().size() > StatisticsUtil.getAutoAnalyzeTableWidthThreshold()) {
- return null;
- }
-
- AnalysisManager analysisManager = Env.getServingEnv().getAnalysisManager();
- TableStatsMeta tblStats = analysisManager.findTableStatsStatus(table.getId());
-
- Map<String, Set<String>> needRunPartitions = null;
- String colNames = jobInfo.colName;
- if (table.needReAnalyzeTable(tblStats)) {
- needRunPartitions = table.findReAnalyzeNeededPartitions();
- } else if (table instanceof OlapTable && tblStats.newPartitionLoaded.get()) {
- OlapTable olapTable = (OlapTable) table;
- needRunPartitions = new HashMap<>();
- Set<String> partitionColumnNames = olapTable.getPartitionInfo().getPartitionColumns().stream()
- .map(Column::getName).collect(Collectors.toSet());
- colNames = partitionColumnNames.stream().collect(Collectors.joining(","));
- Set<String> partitionNames = olapTable.getAllPartitions().stream()
- .map(Partition::getName).collect(Collectors.toSet());
- for (String column : partitionColumnNames) {
- needRunPartitions.put(column, partitionNames);
- }
- }
-
- if (needRunPartitions == null || needRunPartitions.isEmpty()) {
- return null;
- }
- return new AnalysisInfoBuilder(jobInfo).setColName(colNames).setColToPartitions(needRunPartitions).build();
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCollector.java
index 0985b9b2b95..e26d3170178 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCollector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCollector.java
@@ -61,10 +61,6 @@ public abstract class StatisticsCollector extends MasterDaemon {
@VisibleForTesting
protected void createSystemAnalysisJob(AnalysisInfo jobInfo)
throws DdlException {
- if (jobInfo.colToPartitions.isEmpty()) {
- // No statistics need to be collected or updated
- return;
- }
Map<Long, BaseAnalysisTask> analysisTasks = new HashMap<>();
AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
analysisManager.createTaskForEachColumns(jobInfo, analysisTasks, false);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java
new file mode 100644
index 00000000000..73d0d1340ad
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java
@@ -0,0 +1,135 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.statistics;
+
+import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.Table;
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.common.util.MasterDaemon;
+import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.statistics.util.StatisticsUtil;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Queue;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class StatisticsJobAppender extends MasterDaemon {
+
+ private static final Logger LOG = LogManager.getLogger(StatisticsJobAppender.class);
+
+ public static final long INTERVAL = 1000;
+ public static final int JOB_MAP_SIZE = 1000;
+
+ private long currentDbId;
+ private long currentTableId;
+
+ public StatisticsJobAppender() {
+ super("Statistics Job Appender", INTERVAL);
+ }
+
+ @Override
+ protected void runAfterCatalogReady() {
+ if (!StatisticsUtil.enableAutoAnalyze()) {
+ return;
+ }
+ if (!Env.getCurrentEnv().isMaster()) {
+ return;
+ }
+ if (!StatisticsUtil.statsTblAvailable()) {
+ LOG.info("Stats table not available, skip");
+ return;
+ }
+ if (Env.isCheckpointThread()) {
+ return;
+ }
+ appendJobs();
+ }
+
+ protected void appendJobs() {
+ AnalysisManager manager = Env.getCurrentEnv().getAnalysisManager();
+ appendColumnsToJobs(manager.highPriorityColumns, manager.highPriorityJobs);
+ appendColumnsToJobs(manager.midPriorityColumns, manager.midPriorityJobs);
+ appendToLowQueue(manager.lowPriorityJobs);
+ }
+
+ protected void appendColumnsToJobs(Queue<HighPriorityColumn> columnQueue, Map<TableIf, Set<String>> jobsMap) {
+ int size = columnQueue.size();
+ for (int i = 0; i < size; i++) {
+ HighPriorityColumn column = columnQueue.poll();
+ TableIf table = StatisticsUtil.findTable(column.catalogId, column.dbId, column.tblId);
+ synchronized (jobsMap) {
+ // If job map reach the upper limit, stop putting new jobs.
+ if (!jobsMap.containsKey(table) && jobsMap.size() >= JOB_MAP_SIZE) {
+ break;
+ }
+ if (jobsMap.containsKey(table)) {
+ jobsMap.get(table).add(column.colName);
+ } else {
+ jobsMap.put(table, Collections.singleton(column.colName));
+ }
+ }
+ }
+ }
+
+ protected void appendToLowQueue(Map<TableIf, Set<String>> jobsMap) {
+
+ InternalCatalog catalog = Env.getCurrentInternalCatalog();
+ List<Long> sortedDbs = catalog.getDbIds().stream().sorted().collect(Collectors.toList());
+ for (long dbId : sortedDbs) {
+ if (dbId < currentDbId
+ || StatisticConstants.SYSTEM_DBS.contains(catalog.getDbNullable(dbId).getFullName())) {
+ continue;
+ }
+ currentDbId = dbId;
+ Optional<Database> db = catalog.getDb(dbId);
+ List<Table> tables = db.get().getTables().stream()
+ .sorted((t1, t2) -> (int) (t1.getId() - t2.getId())).collect(Collectors.toList());
+ for (Table t : tables) {
+ if (t.getId() <= currentTableId) {
+ continue;
+ }
+ synchronized (jobsMap) {
+ // If job map reach the upper limit, stop putting new jobs.
+ if (!jobsMap.containsKey(t) && jobsMap.size() >= JOB_MAP_SIZE) {
+ return;
+ }
+ Set<String> columns
+ = t.getColumns().stream().filter(c -> !StatisticsUtil.isUnsupportedType(c.getType()))
+ .map(c -> c.getName()).collect(Collectors.toSet());
+ if (jobsMap.containsKey(t)) {
+ jobsMap.get(t).addAll(columns);
+ } else {
+ jobsMap.put(t, columns);
+ }
+ }
+ currentTableId = t.getId();
+ }
+ }
+ // All tables have been processed once, reset for the next loop.
+ currentDbId = 0;
+ currentTableId = 0;
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
deleted file mode 100644
index 678e7580f8e..00000000000
--- a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
+++ /dev/null
@@ -1,560 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.statistics;
-
-import org.apache.doris.catalog.Column;
-import org.apache.doris.catalog.Database;
-import org.apache.doris.catalog.DatabaseIf;
-import org.apache.doris.catalog.Env;
-import org.apache.doris.catalog.EnvFactory;
-import org.apache.doris.catalog.OlapTable;
-import org.apache.doris.catalog.PrimitiveType;
-import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.TableIf;
-import org.apache.doris.catalog.Type;
-import org.apache.doris.catalog.View;
-import org.apache.doris.common.Config;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.common.FeConstants;
-import org.apache.doris.datasource.CatalogIf;
-import org.apache.doris.datasource.InternalCatalog;
-import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod;
-import org.apache.doris.statistics.AnalysisInfo.AnalysisType;
-import org.apache.doris.statistics.AnalysisInfo.JobType;
-import org.apache.doris.statistics.util.StatisticsUtil;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import mockit.Expectations;
-import mockit.Injectable;
-import mockit.Mock;
-import mockit.MockUp;
-import mockit.Mocked;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-import java.time.LocalTime;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-public class StatisticsAutoCollectorTest {
-
- @Test
- public void testAnalyzeAll(@Injectable AnalysisInfo analysisInfo) {
- new MockUp<CatalogIf>() {
- @Mock
- public Collection<DatabaseIf> getAllDbs() {
- Database db1 = new Database(1, FeConstants.INTERNAL_DB_NAME);
- Database db2 = new Database(2, "anyDB");
- List<DatabaseIf> databaseIfs = new ArrayList<>();
- databaseIfs.add(db1);
- databaseIfs.add(db2);
- return databaseIfs;
- }
- };
- new MockUp<StatisticsAutoCollector>() {
- @Mock
- public List<AnalysisInfo> constructAnalysisInfo(DatabaseIf<TableIf> db) {
- return Arrays.asList(analysisInfo, analysisInfo);
- }
-
- int count = 0;
-
- @Mock
- public AnalysisInfo getReAnalyzeRequiredPart(AnalysisInfo jobInfo) {
- return count++ == 0 ? null : jobInfo;
- }
-
- @Mock
- public void createSystemAnalysisJob(AnalysisInfo jobInfo)
- throws DdlException {
-
- }
- };
-
- StatisticsAutoCollector saa = new StatisticsAutoCollector();
- saa.runAfterCatalogReady();
- new Expectations() {
- {
- try {
- saa.createSystemAnalysisJob((AnalysisInfo) any);
- times = 1;
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
- };
- }
-
- @Test
- public void testConstructAnalysisInfo(
- @Injectable OlapTable o2, @Injectable View v) {
- new MockUp<Database>() {
- @Mock
- public List<Table> getTables() {
- List<Table> tableIfs = new ArrayList<>();
- tableIfs.add(o2);
- tableIfs.add(v);
- return tableIfs;
- }
-
- @Mock
- public String getFullName() {
- return "anyDb";
- }
- };
-
- new MockUp<OlapTable>() {
- @Mock
- public String getName() {
- return "anytable";
- }
-
- @Mock
- public List<Column> getSchemaAllIndexes(boolean full) {
- List<Column> columns = new ArrayList<>();
- columns.add(new Column("c1", PrimitiveType.INT));
- columns.add(new Column("c2", PrimitiveType.HLL));
- return columns;
- }
- };
- StatisticsAutoCollector saa = new StatisticsAutoCollector();
- List<AnalysisInfo> analysisInfoList = saa.constructAnalysisInfo(new Database(1, "anydb"));
- Assertions.assertEquals(1, analysisInfoList.size());
- Assertions.assertEquals("c1", analysisInfoList.get(0).colName.split(",")[0]);
- }
-
- @Test
- public void testGetReAnalyzeRequiredPart0() {
-
- TableIf tableIf = new OlapTable();
-
- new MockUp<OlapTable>() {
- @Mock
- protected Map<String, Set<String>> findReAnalyzeNeededPartitions() {
- Set<String> partitionNames = new HashSet<>();
- partitionNames.add("p1");
- partitionNames.add("p2");
- Map<String, Set<String>> map = new HashMap<>();
- map.put("col1", partitionNames);
- return map;
- }
-
- @Mock
- public long getRowCount() {
- return 100;
- }
-
- @Mock
- public List<Column> getBaseSchema() {
- return Lists.newArrayList(new Column("col1", Type.INT), new Column("col2", Type.INT));
- }
- };
-
- new MockUp<StatisticsUtil>() {
- @Mock
- public TableIf findTable(long catalogName, long dbName, long tblName) {
- return tableIf;
- }
- };
- AnalysisInfo analysisInfo = new AnalysisInfoBuilder().setAnalysisMethod(AnalysisMethod.FULL)
- .setColToPartitions(new HashMap<>()).setAnalysisType(
- AnalysisType.FUNDAMENTALS).setColName("col1").setJobType(JobType.SYSTEM).build();
- new MockUp<AnalysisManager>() {
-
- int count = 0;
-
- TableStatsMeta[] tableStatsArr =
- new TableStatsMeta[] {new TableStatsMeta(0, analysisInfo, tableIf),
- new TableStatsMeta(0, analysisInfo, tableIf), null};
-
- {
- tableStatsArr[0].updatedRows.addAndGet(100);
- tableStatsArr[1].updatedRows.addAndGet(0);
- }
-
- @Mock
- public TableStatsMeta findTableStatsStatus(long tblId) {
- return tableStatsArr[count++];
- }
- };
-
- new MockUp<StatisticsAutoCollector>() {
- @Mock
- public AnalysisInfo getAnalysisJobInfo(AnalysisInfo jobInfo, TableIf table,
- Set<String> needRunPartitions) {
- return new AnalysisInfoBuilder().build();
- }
- };
- StatisticsAutoCollector statisticsAutoCollector = new StatisticsAutoCollector();
- AnalysisInfo analysisInfo2 = new AnalysisInfoBuilder()
- .setCatalogId(0)
- .setDBId(0)
- .setTblId(0).build();
- Assertions.assertNotNull(statisticsAutoCollector.getReAnalyzeRequiredPart(analysisInfo2));
- // uncomment it when updatedRows gets ready
- // Assertions.assertNull(statisticsAutoCollector.getReAnalyzeRequiredPart(analysisInfo2));
- Assertions.assertNotNull(statisticsAutoCollector.getReAnalyzeRequiredPart(analysisInfo2));
- }
-
- @Test
- public void testSkipWideTable() {
-
- TableIf tableIf = new OlapTable();
-
- new MockUp<OlapTable>() {
- @Mock
- public List<Column> getBaseSchema() {
- return Lists.newArrayList(new Column("col1", Type.INT), new Column("col2", Type.INT));
- }
- };
-
- new MockUp<StatisticsUtil>() {
- int count = 0;
- int [] thresholds = {1, 10};
- @Mock
- public TableIf findTable(long catalogName, long dbName, long tblName) {
- return tableIf;
- }
-
- @Mock
- public int getAutoAnalyzeTableWidthThreshold() {
- return thresholds[count++];
- }
- };
-
- new MockUp<OlapTable>() {
- @Mock
- public Map<String, Set<String>> findReAnalyzeNeededPartitions() {
- HashMap<String, Set<String>> ret = Maps.newHashMap();
- ret.put("key1", Sets.newHashSet());
- return ret;
- }
- };
-
- AnalysisInfo analysisInfo = new AnalysisInfoBuilder().build();
- StatisticsAutoCollector statisticsAutoCollector = new StatisticsAutoCollector();
- Assertions.assertNull(statisticsAutoCollector.getReAnalyzeRequiredPart(analysisInfo));
- Assertions.assertNotNull(statisticsAutoCollector.getReAnalyzeRequiredPart(analysisInfo));
- }
-
- @Test
- public void testLoop() {
- AtomicBoolean timeChecked = new AtomicBoolean();
- AtomicBoolean switchChecked = new AtomicBoolean();
- new MockUp<StatisticsUtil>() {
-
- @Mock
- public boolean inAnalyzeTime(LocalTime now) {
- timeChecked.set(true);
- return true;
- }
-
- @Mock
- public boolean enableAutoAnalyze() {
- switchChecked.set(true);
- return true;
- }
- };
- StatisticsAutoCollector autoCollector = new StatisticsAutoCollector();
- autoCollector.collect();
- Assertions.assertTrue(timeChecked.get() && switchChecked.get());
-
- }
-
- @Test
- public void checkAvailableThread() {
- StatisticsAutoCollector autoCollector = new StatisticsAutoCollector();
- Assertions.assertEquals(Config.auto_analyze_simultaneously_running_task_num,
- autoCollector.analysisTaskExecutor.executors.getMaximumPoolSize());
- }
-
- @Test
- public void testSkip(@Mocked OlapTable olapTable, @Mocked TableStatsMeta stats, @Mocked TableIf anyOtherTable) {
- new MockUp<OlapTable>() {
-
- @Mock
- public long getDataSize(boolean singleReplica) {
- return StatisticsUtil.getHugeTableLowerBoundSizeInBytes() * 5 + 1000000000;
- }
- };
-
- new MockUp<AnalysisManager>() {
-
- @Mock
- public TableStatsMeta findTableStatsStatus(long tblId) {
- return stats;
- }
- };
- // A very huge table has been updated recently, so we should skip it this time
- stats.updatedTime = System.currentTimeMillis() - 1000;
- stats.newPartitionLoaded = new AtomicBoolean();
- stats.newPartitionLoaded.set(true);
- StatisticsAutoCollector autoCollector = new StatisticsAutoCollector();
- // Test new partition loaded data for the first time. Not skip.
- Assertions.assertFalse(autoCollector.skip(olapTable));
- stats.newPartitionLoaded.set(false);
- // Assertions.assertTrue(autoCollector.skip(olapTable));
- // The update of this huge table is long time ago, so we shouldn't skip it this time
- stats.updatedTime = System.currentTimeMillis()
- - StatisticsUtil.getHugeTableAutoAnalyzeIntervalInMillis() - 10000;
- Assertions.assertFalse(autoCollector.skip(olapTable));
- new MockUp<AnalysisManager>() {
-
- @Mock
- public TableStatsMeta findTableStatsStatus(long tblId) {
- return null;
- }
- };
- // can't find table stats meta, which means this table never get analyzed, so we shouldn't skip it this time
- Assertions.assertFalse(autoCollector.skip(olapTable));
- new MockUp<AnalysisManager>() {
-
- @Mock
- public TableStatsMeta findTableStatsStatus(long tblId) {
- return stats;
- }
- };
- stats.userInjected = true;
- Assertions.assertTrue(autoCollector.skip(olapTable));
- // this is not olap table nor external table, so we should skip it this time
- Assertions.assertTrue(autoCollector.skip(anyOtherTable));
- }
-
- // For small table, use full
- @Test
- public void testCreateAnalyzeJobForTbl1(
- @Injectable OlapTable t1,
- @Injectable Database db
- ) throws Exception {
- new MockUp<Database>() {
-
- @Mock
- public CatalogIf getCatalog() {
- return Env.getCurrentInternalCatalog();
- }
-
- @Mock
- public long getId() {
- return 0;
- }
- };
- new MockUp<OlapTable>() {
-
- int count = 0;
-
- @Mock
- public List<Column> getBaseSchema() {
- return Lists.newArrayList(new Column("test", PrimitiveType.INT));
- }
-
- @Mock
- public long getDataSize(boolean singleReplica) {
- return StatisticsUtil.getHugeTableLowerBoundSizeInBytes() - 1;
- }
-
- @Mock
- public BaseAnalysisTask createAnalysisTask(AnalysisInfo info) {
- return new OlapAnalysisTask(info);
- }
-
- @Mock
- public List<Long> getMvColumnIndexIds(String columnName) {
- ArrayList<Long> objects = new ArrayList<>();
- objects.add(-1L);
- return objects;
- }
- };
-
- new MockUp<StatisticsUtil>() {
- @Mock
- public TableIf findTable(long catalogId, long dbId, long tblId) {
- return t1;
- }
- };
-
- StatisticsAutoCollector sac = new StatisticsAutoCollector();
- List<AnalysisInfo> jobInfos = new ArrayList<>();
- sac.createAnalyzeJobForTbl(db, jobInfos, t1);
- AnalysisInfo jobInfo = jobInfos.get(0);
- Map<String, Set<String>> colToPartitions = new HashMap<>();
- colToPartitions.put("test", new HashSet<String>() {
- {
- add("p1");
- }
- });
- jobInfo = new AnalysisInfoBuilder(jobInfo).setColToPartitions(colToPartitions).build();
- Map<Long, BaseAnalysisTask> analysisTasks = new HashMap<>();
- AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
- analysisManager.createTaskForEachColumns(jobInfo, analysisTasks, false);
- Assertions.assertEquals(1, analysisTasks.size());
- for (BaseAnalysisTask task : analysisTasks.values()) {
- Assertions.assertNull(task.getTableSample());
- }
- }
-
- // for big table, use sample
- @Test
- public void testCreateAnalyzeJobForTbl2(
- @Injectable OlapTable t1,
- @Injectable Database db
- ) throws Exception {
- new MockUp<Database>() {
-
- @Mock
- public CatalogIf getCatalog() {
- return Env.getCurrentInternalCatalog();
- }
-
- @Mock
- public long getId() {
- return 0;
- }
- };
- new MockUp<OlapTable>() {
-
- int count = 0;
-
- @Mock
- public List<Column> getBaseSchema() {
- return Lists.newArrayList(new Column("test", PrimitiveType.INT));
- }
-
- @Mock
- public long getDataSize(boolean singleReplica) {
- return StatisticsUtil.getHugeTableLowerBoundSizeInBytes() * 2;
- }
-
- @Mock
- public BaseAnalysisTask createAnalysisTask(AnalysisInfo info) {
- return new OlapAnalysisTask(info);
- }
-
- @Mock
- public List<Long> getMvColumnIndexIds(String columnName) {
- ArrayList<Long> objects = new ArrayList<>();
- objects.add(-1L);
- return objects;
- }
- };
-
- new MockUp<StatisticsUtil>() {
- @Mock
- public TableIf findTable(long catalogId, long dbId, long tblId) {
- return t1;
- }
- };
-
- StatisticsAutoCollector sac = new StatisticsAutoCollector();
- List<AnalysisInfo> jobInfos = new ArrayList<>();
- sac.createAnalyzeJobForTbl(db, jobInfos, t1);
- AnalysisInfo jobInfo = jobInfos.get(0);
- Map<String, Set<String>> colToPartitions = new HashMap<>();
- colToPartitions.put("test", new HashSet<String>() {
- {
- add("p1");
- }
- });
- jobInfo = new AnalysisInfoBuilder(jobInfo).setColToPartitions(colToPartitions).build();
- Map<Long, BaseAnalysisTask> analysisTasks = new HashMap<>();
- AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
- analysisManager.createTaskForEachColumns(jobInfo, analysisTasks, false);
- Assertions.assertEquals(1, analysisTasks.size());
- for (BaseAnalysisTask task : analysisTasks.values()) {
- Assertions.assertNotNull(task.getTableSample());
- }
- }
-
- @Test
- public void testDisableAuto1() throws Exception {
- InternalCatalog catalog1 = EnvFactory.getInstance().createInternalCatalog();
- List<CatalogIf> catalogs = Lists.newArrayList();
- catalogs.add(catalog1);
-
- new MockUp<StatisticsAutoCollector>() {
- @Mock
- public List<CatalogIf> getCatalogsInOrder() {
- return catalogs;
- }
-
- @Mock
- protected boolean canCollect() {
- return false;
- }
-
- };
-
- StatisticsAutoCollector sac = new StatisticsAutoCollector();
- new Expectations(catalog1) {{
- catalog1.enableAutoAnalyze();
- times = 0;
- }};
-
- sac.analyzeAll();
- }
-
- @Test
- public void testDisableAuto2() throws Exception {
- InternalCatalog catalog1 = EnvFactory.getInstance().createInternalCatalog();
- List<CatalogIf> catalogs = Lists.newArrayList();
- catalogs.add(catalog1);
-
- Database db1 = new Database();
- List<DatabaseIf<? extends TableIf>> dbs = Lists.newArrayList();
- dbs.add(db1);
-
- new MockUp<StatisticsAutoCollector>() {
- int count = 0;
- boolean[] canCollectReturn = {true, false};
- @Mock
- public List<CatalogIf> getCatalogsInOrder() {
- return catalogs;
- }
-
- @Mock
- public List<DatabaseIf<? extends TableIf>> getDatabasesInOrder(CatalogIf<DatabaseIf> catalog) {
- return dbs;
- }
-
- @Mock
- protected boolean canCollect() {
- return canCollectReturn[count++];
- }
-
- };
-
- StatisticsAutoCollector sac = new StatisticsAutoCollector();
- new Expectations(catalog1, db1) {{
- catalog1.enableAutoAnalyze();
- result = true;
- times = 1;
- db1.getFullName();
- times = 0;
- }};
-
- sac.analyzeAll();
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 09/09: refactor2 (#32278)
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch high-priority-column
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 6ad122238841c917deaad94f26b2214b4df48516
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Fri Mar 15 11:58:38 2024 +0800
refactor2 (#32278)
---
.../doris/nereids/jobs/executor/Rewriter.java | 4 +--
...umnCollector.java => QueryColumnCollector.java} | 32 +++++++++++-----------
.../apache/doris/statistics/AnalysisManager.java | 6 ++--
3 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index 9977a36879c..1818529302b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -32,7 +32,7 @@ import org.apache.doris.nereids.rules.expression.CheckLegalityAfterRewrite;
import org.apache.doris.nereids.rules.expression.ExpressionNormalization;
import org.apache.doris.nereids.rules.expression.ExpressionOptimization;
import org.apache.doris.nereids.rules.expression.ExpressionRewrite;
-import org.apache.doris.nereids.rules.expression.HighPriorityColumnCollector;
+import org.apache.doris.nereids.rules.expression.QueryColumnCollector;
import org.apache.doris.nereids.rules.rewrite.AddDefaultLimit;
import org.apache.doris.nereids.rules.rewrite.AdjustConjunctsReturnType;
import org.apache.doris.nereids.rules.rewrite.AdjustNullable;
@@ -410,7 +410,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
new CollectProjectAboveConsumer()
)
),
- topic("Collect used column", custom(RuleType.COLLECT_COLUMNS, HighPriorityColumnCollector::new))
+ topic("Collect used column", custom(RuleType.COLLECT_COLUMNS, QueryColumnCollector::new))
);
private static final List<RewriteJob> WHOLE_TREE_REWRITE_JOBS
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/HighPriorityColumnCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/QueryColumnCollector.java
similarity index 89%
rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/HighPriorityColumnCollector.java
rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/QueryColumnCollector.java
index e20363b1202..ebf361de1d3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/HighPriorityColumnCollector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/QueryColumnCollector.java
@@ -21,7 +21,7 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.nereids.jobs.JobContext;
-import org.apache.doris.nereids.rules.expression.HighPriorityColumnCollector.CollectorContext;
+import org.apache.doris.nereids.rules.expression.QueryColumnCollector.CollectorContext;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
@@ -51,9 +51,9 @@ import java.util.Set;
import java.util.stream.Collectors;
/**
- * Used to collect High priority column.
+ * Used to collect query column.
*/
-public class HighPriorityColumnCollector extends DefaultPlanRewriter<CollectorContext> implements CustomRewriter {
+public class QueryColumnCollector extends DefaultPlanRewriter<CollectorContext> implements CustomRewriter {
@Override
public Plan rewriteRoot(Plan plan, JobContext jobContext) {
@@ -64,10 +64,10 @@ public class HighPriorityColumnCollector extends DefaultPlanRewriter<CollectorCo
CollectorContext context = new CollectorContext();
plan.accept(this, context);
if (StatisticsUtil.enableAutoAnalyze()) {
- context.queried.removeAll(context.usedInPredicate);
+ context.midPriority.removeAll(context.highPriority);
AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
- analysisManager.updateColumnUsedInPredicate(context.usedInPredicate);
- analysisManager.updateQueriedColumn(context.queried);
+ analysisManager.updateHighPriorityColumn(context.highPriority);
+ analysisManager.updateMidPriorityColumn(context.midPriority);
}
return plan;
}
@@ -78,9 +78,9 @@ public class HighPriorityColumnCollector extends DefaultPlanRewriter<CollectorCo
public static class CollectorContext {
public Map<Slot/*project output column*/, NamedExpression/*Actual project expr*/> projects = new HashMap<>();
- public Set<Slot> usedInPredicate = new HashSet<>();
+ public Set<Slot> highPriority = new HashSet<>();
- public Set<Slot> queried = new HashSet<>();
+ public Set<Slot> midPriority = new HashSet<>();
}
@Override
@@ -103,7 +103,7 @@ public class HighPriorityColumnCollector extends DefaultPlanRewriter<CollectorCo
List<Slot> outputOfScan = scan.getOutput();
for (Slot slot : outputOfScan) {
if (!allUsed.contains(slot)) {
- context.queried.remove(slot);
+ context.midPriority.remove(slot);
}
}
}
@@ -114,7 +114,7 @@ public class HighPriorityColumnCollector extends DefaultPlanRewriter<CollectorCo
public Plan visitLogicalJoin(LogicalJoin<? extends Plan, ? extends Plan> join, CollectorContext context) {
join.child(0).accept(this, context);
join.child(1).accept(this, context);
- context.usedInPredicate.addAll(
+ context.highPriority.addAll(
(join.isMarkJoin() ? join.getLeftConditionSlot() : join.getConditionSlot())
.stream().flatMap(s -> backtrace(s, context).stream())
.collect(Collectors.toSet())
@@ -125,7 +125,7 @@ public class HighPriorityColumnCollector extends DefaultPlanRewriter<CollectorCo
@Override
public Plan visitLogicalAggregate(LogicalAggregate<? extends Plan> aggregate, CollectorContext context) {
aggregate.child(0).accept(this, context);
- context.usedInPredicate.addAll(aggregate.getGroupByExpressions()
+ context.highPriority.addAll(aggregate.getGroupByExpressions()
.stream()
.flatMap(e -> e.<Set<SlotReference>>collect(n -> n instanceof SlotReference).stream())
.flatMap(s -> backtrace(s, context).stream())
@@ -136,7 +136,7 @@ public class HighPriorityColumnCollector extends DefaultPlanRewriter<CollectorCo
@Override
public Plan visitLogicalHaving(LogicalHaving<? extends Plan> having, CollectorContext context) {
having.child(0).accept(this, context);
- context.usedInPredicate.addAll(
+ context.highPriority.addAll(
having.getExpressions().stream()
.flatMap(e -> e.<Set<SlotReference>>collect(n -> n instanceof SlotReference).stream())
.flatMap(s -> backtrace(s, context).stream())
@@ -147,21 +147,21 @@ public class HighPriorityColumnCollector extends DefaultPlanRewriter<CollectorCo
@Override
public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, CollectorContext context) {
List<Slot> slots = olapScan.getOutput();
- context.queried.addAll(slots);
+ context.midPriority.addAll(slots);
return olapScan;
}
@Override
public Plan visitLogicalFileScan(LogicalFileScan fileScan, CollectorContext context) {
List<Slot> slots = fileScan.getOutput();
- context.queried.addAll(slots);
+ context.midPriority.addAll(slots);
return fileScan;
}
@Override
public Plan visitLogicalFilter(LogicalFilter<? extends Plan> filter, CollectorContext context) {
filter.child(0).accept(this, context);
- context.usedInPredicate.addAll(filter
+ context.highPriority.addAll(filter
.getExpressions()
.stream()
.flatMap(e -> e.<Set<SlotReference>>collect(n -> n instanceof SlotReference).stream())
@@ -173,7 +173,7 @@ public class HighPriorityColumnCollector extends DefaultPlanRewriter<CollectorCo
@Override
public Plan visitLogicalWindow(LogicalWindow<? extends Plan> window, CollectorContext context) {
window.child(0).accept(this, context);
- context.usedInPredicate.addAll(window
+ context.highPriority.addAll(window
.getWindowExpressions()
.stream()
.flatMap(e -> e.<Set<SlotReference>>collect(n -> n instanceof SlotReference).stream())
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index 8a30aec9e60..d795b64fbaa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -111,7 +111,7 @@ public class AnalysisManager implements Writable {
private static final Logger LOG = LogManager.getLogger(AnalysisManager.class);
- private static final int COLUMN_QUEUE_SIZE = 1000;
+ public static final int COLUMN_QUEUE_SIZE = 1000;
public final Queue<QueryColumn> highPriorityColumns = new ArrayBlockingQueue<>(COLUMN_QUEUE_SIZE);
public final Queue<QueryColumn> midPriorityColumns = new ArrayBlockingQueue<>(COLUMN_QUEUE_SIZE);
public final Map<TableName, Set<String>> highPriorityJobs = new LinkedHashMap<>();
@@ -1152,11 +1152,11 @@ public class AnalysisManager implements Writable {
}
- public void updateColumnUsedInPredicate(Set<Slot> slotReferences) {
+ public void updateHighPriorityColumn(Set<Slot> slotReferences) {
updateColumn(slotReferences, highPriorityColumns);
}
- public void updateQueriedColumn(Collection<Slot> slotReferences) {
+ public void updateMidPriorityColumn(Collection<Slot> slotReferences) {
updateColumn(slotReferences, midPriorityColumns);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org