You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by li...@apache.org on 2021/12/31 03:35:34 UTC
[incubator-doris] branch vectorized updated (164b274 -> e0cadc4)
This is an automated email from the ASF dual-hosted git repository.
lihaopeng pushed a change to branch vectorized
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git.
from 164b274 [revert] "[improvement](bdbje) clean too many bdbje log (#7273)" (#7312)
add 8660bf6 [fix](select join) Make selected slotRef nullable when slotRef is from nullable tuple in outer join sql block (#7290)
add 62d1206 [feature](udf) make orthogonal bitmap udaf as build in functions (#7211)
add 03ad8c1 [fix](load) Fix bug that show load may be blocked (#7254)
add 5e32ae3 [improvement](cache) Optimize sql cache (#7231)
add 3b10002 [community][typo](github) modify PR template (#7310)
add 868281f [docs] update data-model-rollup.md (#7321)
add 2ae9c41 [fix](lateral view)(subquery) Fix column materialization error (#7330)
add 10ccada [fix](forward) Avoid endless forward execution (#7335)
add 6f91741 [Bug]Fix BE coredump when manual compaction task is triggered (#7260)
add be0cf51 [docs] add java formatter in doc (#7306)
add b080e79 [community](github) add more content of gitignore file (#7307)
add dc281eb [fix](routine load) fix bug that can not read image when using keyword STREAM (#7323)
add db57c42 [improvement](compaction)(tablet repair) Add missing rowsets in compaction status url and support force dropping redundant replica (#7283)
add ac739fe [refactor] modify the control flow code to improve code readability (#7302)
add 80c11da [refactor] modify the implements of Tuple & RowBatch (#7319)
add 568f661 [deps](log4j) upgrade log4j (#7364)
add 5745adb [improvement](reader) optimize for single rowset reading (#7351)
add e0889ae [typo](load) correct the error of ‘EtlJobMgr::get_job_status’ function (#7353)
add ef2ea18 [docs] Improve the chapter on debugging FE in doc. (#7309)
add 414c5a8 [fix] LRUCache::prune_if may not remove all the entries matching the predicate (#7383)
add a6a584a [doc] update the compilation.md (#7350)
add 4e02109 [refactor][fix](constants-fold) Refactor the code of fold constant mgr and fix some undefined behavior and mem leak (#7373)
add d9c927f [improvement](log)(schema change) Add a clear memory description in the log (#7378)
add e64da03 [deps](log4j) Upgrade log4j 2 to 2.16.0 (#7394)
add 926540c [feature] Support return bitmp/hll data in select statement (#7276)
add 382351b [fix](ut) Fix run fe ut failed, be ut memory leak and build thirdparty failed (#7377)
add c8bc0cf [chore][community](github) Remove travis and add github action (#7380)
add 549e849 [improvement](flink-connector) DataSourceFunction read doris supports parallel (#7232)
add 8552194 [refactor](olap-scan-node) Refactor olap scannode (#7131)
add 4afdcdb [performance](reader) Opt the unique reader to reduce unnecessary compare and function call (#7348)
add 6ede693 [fix](insert) modify code logic of InsertStmt (#7360)
add 2e334d0 [docs](sql-block-rule) modify document of sql block rule (#7370)
add 2b90967 [fix][refactor](broker load) refactor the scheduling logic of broker load (#7371)
add 6dd312b [docs](website) develop the caseList component (#7402)
add 5fed8a9 [docs](flink-connector) Add instructions for flink doris connector (#7384)
add 0499b22 [feat](lateral-view) Support execution of lateral view stmt (#7255)
add c873c8c [fix](lateral view)(subquery) Forbidden directly AGG/SORT on lateral view (#7337)
add 7d4da7a [fix](rpc) fix BE crash in SendRpcResponse when high concurrency (#7413)
add 06c38ce [enhancement] Make concurrent_number for routine load task can be larger than be num (#7386)
add e9536a8 [deps](cyrus_sasl) Add -fPIC for cyrus_sasl (#7408)
add f6e598d Revert "[improvement](reader) optimize for single rowset reading (#7351)" (#7427)
add e74e55d [docs] Fix typos (#7404)
add 6c320df [community](github) Add .asf.yaml (#7431)
add 7a1bb5b log4j upgrade to 2.17.0 (#7440)
add 2d72c03 [deps](openssl) upgrade openssl to 1.1.1m (#7446)
add 30db2cd [fix](cache) Some view stmt cannot be obtained when view in the subquery and add cache key UT (#7375)
add 998489a [fix](sql-block-rule) move sql block rule check from ConnectProcessor to StmtExecutor (#7407)
add 560c8b8 [enhancement] Remove the two lines of duplicate import. (#7331) (#7332)
add 695eca8 [docs] add bloomfilter index doc (#7318)
add e904960 [fix](flink-connector) Connector should visit the surviving BE nodes (#7435)
add 97749ed [community][chore] Modify .asf.yaml and fix BE build warning (#7439)
add 2ab3a66 [docs][community] Remove articles (#7449)
add 20ef8a6 [feature-wip](remote storage)(step1) use a struct instead of string for parameter path, add basic remote method (#7098)
add a8c444d [fix](sql-rewrite) Rewrite Bigint SlotRef to compare DecimalLiteral in Binary predicate (#7265)
add 889e33d [docs](seatunnel) Seatunnel Supports Doris connector (#7453)
add 3128c7c [fix](ut) fix testPartitionRebalancer ut (#7468)
add ff5a0e9 [improvement](planner) make BinaryPredicate do not cast date to datetime/varchar (#7045)
add 3ba6dcf [fix](function) fix round function for inaccuracy (#7421)
add 6d1cf59 [fix] DCHECK fail at BitmapValue getSizeInBytes (#7430)
add bfa6bc3 [fix](function) fix aggregate function min() at type varchar (#7437)
add c596b03 [docs](docker) Add document of docker dev (#7447)
add b4ce189 [improvement](flink-connector) flush data without multi httpclients (#7329) (#7450)
add a8a5c0a [improvement](load) memory usage optimization for load job (#7454)
add 43ed54f [docs] The name of hidden column is incorrect in batch-delete-manual.md(#7465) (#7466)
add 91332fa [fix](reader) fix logic error for Tablet::capture_rs_readers (#7469)
add 2347f12 [fix](fe-ut) Fix NPE when start FE in unit test. (#7471)
add f33d1f7 [fix](ut) Fix FE ut SelectStmtTest.testDeduplicateOrs (#7481)
add 4ed1846 [fix](ut) Fix BE broker scanner unit test bug (#7486)
add fe1d0c1 [fix](materialized-view)(planner) fix mv rewrite bug (#7362)
add 0c15473 [feature](function) support bitmap_union/intersect have more columns parameters (#7379)
add 98551f8 [fix](grouping-set) Grouping set clause act wrong for function expr in view (#7410) (#7411)
add ca97535 [docs](executor) correct some be error code (#7460)
add 43e9318 [chore](docker) Add clang11 in docker dev image (#7470)
add ab60c5e [fix](spark-load) fix Roaring64Map big-endian read/write in de/serialization (#7480)
add 755e069 [feature](broker) support ks3 for kmr in ksyun (#7484)
add 80587e7 [improvement](spark-connector)(flink-connector) Modify the max num of batch written by Spark/Flink connector each time. (#7485)
add 6e052f4 [Doc][Website] blogs are sorted by date (#7491)
add 07e2acb [feature] Suport national secret (national commercial password) algorithm SM3/SM4 (#7464)
add 3454735 [fix](balance) fix partition rebalance bug (#7213)
add 9fb8900 [revert] part of "[improvement](planner) make BinaryPredicate do not cast date to datetime/varchar (#7045)" (#7501)
add a2d6e6e [improvement](config) Modify default value of some brpc config (#7493)
add 3a5de97 [Feature](Partition pruning) Implement V2 version of partition prune. (#7434)
add e933607 Revert "[improvement](planner) make BinaryPredicate do not cast date to datetime/varchar (#7045)" (#7517)
add 2872dbf [refactor] Standardize the writing of pom files, prepare for deployment to maven (#7477)
add bc4ceec [improvement] optimize java cmd find (#7428)
add 85c30fc [deps] Upgrade Log4j to 2.7.1 to solve the CVE-2021-44832 security vulnerability (#7536)
add 0894848 fix having clause constant folding (#7507)
add dc9cd34 [docs] Add user manual for hdfs load and transaction. (#7497)
add 4d01219 [fix](lower_case_table_names) Fix the bug of case-sensitive aliases in the query when lower_case_table_names=1 is set (#7495)
add 7357089 [fix] change percentile_approx return from nan to null (#7512)
add d88711a [fix] fix TableRef.java checkstyle failed (#7538)
add 8da2e8b [fix](cache) Int overflow causes the wrong latest table to be obtained (#7533)
add 723ee84 [feature] (planner) InferPredicate (#7096)
add 7903e64 [Bug](partition pruning v2) Fix NPE when calling `Analyzer.getContext()` in partition pruning related logic. (#7542)
new e0cadc4 [Vectorized Exec Engine] Support Vectorized Exec Engine In Doris
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.asf.yaml | 38 +
.github/PULL_REQUEST_TEMPLATE.md | 4 +-
.github/workflows/build-extension.yml | 74 +
.github/workflows/clang-format.yml | 6 +-
.gitignore | 20 +-
.travis.yml | 56 -
be/CMakeLists.txt | 16 +
be/src/agent/task_worker_pool.cpp | 6 +-
be/src/common/config.h | 14 +-
be/src/common/daemon.cpp | 2 +
be/src/common/utils.h | 10 +
be/src/env/CMakeLists.txt | 2 +
be/src/env/env.cpp | 50 +
be/src/env/env.h | 121 +-
be/src/env/env_posix.cpp | 462 +++---
be/src/env/env_posix.h | 101 ++
be/src/env/env_remote.cpp | 357 +++++
be/src/env/env_remote.h | 107 ++
be/src/exec/CMakeLists.txt | 1 +
be/src/exec/base_scanner.h | 2 +-
be/src/exec/blocking_join_node.cpp | 3 +-
be/src/exec/broker_scan_node.cpp | 12 +-
be/src/exec/broker_scanner.cpp | 7 +-
be/src/exec/broker_scanner.h | 2 +-
be/src/exec/data_sink.cpp | 15 +-
be/src/exec/data_sink.h | 11 +-
be/src/exec/es/es_scroll_parser.cpp | 381 ++++-
be/src/exec/es/es_scroll_parser.h | 8 +
be/src/exec/es_http_scan_node.cpp | 15 +-
be/src/exec/es_http_scan_node.h | 45 +-
be/src/exec/es_http_scanner.h | 2 +-
be/src/exec/exec_node.cpp | 161 +-
be/src/exec/exec_node.h | 21 +-
be/src/exec/hdfs_writer.cpp | 7 +
be/src/exec/json_scanner.cpp | 7 +-
be/src/exec/json_scanner.h | 2 +-
be/src/exec/mysql_scan_node.h | 2 +-
be/src/exec/odbc_scan_node.cpp | 12 +-
be/src/exec/odbc_scan_node.h | 12 +-
be/src/exec/olap_scan_node.cpp | 60 +-
be/src/exec/olap_scan_node.h | 31 +-
be/src/exec/olap_scanner.cpp | 8 +-
be/src/exec/olap_scanner.h | 2 -
be/src/exec/orc_scanner.cpp | 9 +-
be/src/exec/orc_scanner.h | 2 +-
be/src/exec/parquet_scanner.cpp | 7 +-
be/src/exec/parquet_scanner.h | 2 +-
be/src/exec/parquet_writer.cpp | 38 +-
be/src/exec/parquet_writer.h | 4 +-
be/src/exec/partitioned_aggregation_node.cc | 2 +-
be/src/exec/scan_node.h | 7 +-
be/src/exec/schema_scan_node.h | 2 +-
be/src/exec/table_function_node.cpp | 340 ++++
be/src/exec/table_function_node.h | 77 +
be/src/exec/tablet_info.cpp | 224 +++
be/src/exec/tablet_info.h | 100 ++
be/src/exec/tablet_sink.cpp | 72 +-
be/src/exec/tablet_sink.h | 6 +-
be/src/exec/text_converter.cpp | 22 +
be/src/exec/text_converter.h | 7 +-
be/src/exec/text_converter.hpp | 133 +-
be/src/exprs/CMakeLists.txt | 8 +-
be/src/exprs/aggregate_functions.cpp | 26 +-
be/src/exprs/bitmap_function.cpp | 364 ++++-
be/src/exprs/bitmap_function.h | 42 +-
be/src/exprs/encryption_functions.cpp | 179 ++-
be/src/exprs/encryption_functions.h | 42 +-
be/src/exprs/math_functions.cpp | 8 +-
be/src/exprs/math_functions.h | 2 +-
be/src/exprs/runtime_filter.cpp | 34 +
be/src/exprs/runtime_filter.h | 1 +
be/src/exprs/runtime_filter_slots.h | 38 +-
be/src/exprs/scalar_fn_call.cpp | 53 +-
be/src/exprs/scalar_fn_call.h | 5 +
be/src/exprs/slot_ref.cpp | 5 +-
.../exprs/table_function/dummy_table_functions.cpp | 49 +
.../exprs/table_function/dummy_table_functions.h | 47 +
be/src/exprs/table_function/explode_bitmap.cpp | 124 ++
be/src/exprs/table_function/explode_bitmap.h | 56 +
be/src/exprs/table_function/explode_json_array.cpp | 202 +++
be/src/exprs/table_function/explode_json_array.h | 130 ++
be/src/exprs/table_function/explode_split.cpp | 115 ++
be/src/exprs/table_function/explode_split.h | 56 +
be/src/exprs/table_function/table_function.h | 65 +
.../table_function/table_function_factory.cpp | 48 +
.../exprs/table_function/table_function_factory.h | 36 +
be/src/exprs/v_string_functions.h | 219 +++
be/src/http/CMakeLists.txt | 4 +-
be/src/http/action/compaction_action.cpp | 23 +
be/src/http/action/compaction_action.h | 2 +
be/src/olap/aggregate_func.h | 1 +
be/src/olap/base_tablet.cpp | 14 +-
be/src/olap/base_tablet.h | 8 +-
be/src/olap/block_column_predicate.cpp | 114 ++
be/src/olap/block_column_predicate.h | 26 +
be/src/olap/collect_iterator.cpp | 30 +-
be/src/olap/collect_iterator.h | 17 +-
be/src/olap/column_predicate.h | 13 +
be/src/olap/compaction.cpp | 2 +-
be/src/olap/comparison_predicate.cpp | 163 ++
be/src/olap/comparison_predicate.h | 4 +
be/src/olap/data_dir.cpp | 331 ++--
be/src/olap/data_dir.h | 25 +-
be/src/olap/delta_writer.cpp | 2 +-
be/src/olap/fs/CMakeLists.txt | 1 +
be/src/olap/fs/block_manager.h | 17 +-
be/src/olap/fs/file_block_manager.cpp | 87 +-
be/src/olap/fs/file_block_manager.h | 17 +-
be/src/olap/fs/fs_util.cpp | 30 +-
be/src/olap/fs/fs_util.h | 6 +-
be/src/olap/fs/remote_block_manager.cpp | 339 ++++
be/src/olap/fs/remote_block_manager.h | 70 +
be/src/olap/generic_iterators.cpp | 62 +-
be/src/olap/generic_iterators.h | 4 +-
be/src/olap/iterators.h | 6 +-
be/src/olap/lru_cache.cpp | 46 +-
be/src/olap/lru_cache.h | 10 +-
be/src/olap/null_predicate.cpp | 43 +
be/src/olap/null_predicate.h | 8 +
be/src/olap/olap_common.h | 3 +-
be/src/olap/olap_define.h | 1 +
be/src/olap/olap_server.cpp | 10 +-
be/src/olap/options.h | 1 +
be/src/olap/push_handler.cpp | 12 +-
be/src/olap/push_handler.h | 4 +-
be/src/olap/reader.cpp | 17 +-
be/src/olap/reader.h | 24 +-
be/src/olap/row_block2.cpp | 208 +++
be/src/olap/row_block2.h | 5 +
be/src/olap/rowset/alpha_rowset.cpp | 12 +-
be/src/olap/rowset/alpha_rowset.h | 4 +-
be/src/olap/rowset/alpha_rowset_reader.h | 4 +
be/src/olap/rowset/alpha_rowset_writer.cpp | 10 +-
be/src/olap/rowset/beta_rowset.cpp | 106 +-
be/src/olap/rowset/beta_rowset.h | 8 +-
be/src/olap/rowset/beta_rowset_reader.cpp | 42 +-
be/src/olap/rowset/beta_rowset_reader.h | 1 +
be/src/olap/rowset/beta_rowset_writer.cpp | 21 +-
be/src/olap/rowset/rowset.cpp | 4 +-
be/src/olap/rowset/rowset.h | 11 +-
be/src/olap/rowset/rowset_converter.cpp | 14 +-
be/src/olap/rowset/rowset_converter.h | 8 +-
be/src/olap/rowset/rowset_factory.cpp | 6 +-
be/src/olap/rowset/rowset_factory.h | 2 +-
be/src/olap/rowset/rowset_reader.h | 8 +
be/src/olap/rowset/rowset_writer_context.h | 3 +-
be/src/olap/rowset/segment_group.cpp | 4 +-
be/src/olap/rowset/segment_v2/binary_dict_page.cpp | 52 +-
be/src/olap/rowset/segment_v2/binary_dict_page.h | 9 +
be/src/olap/rowset/segment_v2/binary_plain_page.h | 31 +
be/src/olap/rowset/segment_v2/binary_prefix_page.h | 4 +
.../olap/rowset/segment_v2/bitmap_index_reader.cpp | 4 +-
.../olap/rowset/segment_v2/bitmap_index_reader.h | 6 +-
be/src/olap/rowset/segment_v2/bitshuffle_page.h | 48 +
.../segment_v2/bloom_filter_index_reader.cpp | 2 +-
.../rowset/segment_v2/bloom_filter_index_reader.h | 6 +-
be/src/olap/rowset/segment_v2/column_reader.cpp | 131 +-
be/src/olap/rowset/segment_v2/column_reader.h | 26 +-
.../rowset/segment_v2/empty_segment_iterator.cpp | 6 +-
.../rowset/segment_v2/empty_segment_iterator.h | 3 +-
.../rowset/segment_v2/frame_of_reference_page.h | 4 +
.../rowset/segment_v2/indexed_column_reader.cpp | 4 +-
.../olap/rowset/segment_v2/indexed_column_reader.h | 13 +-
.../olap/rowset/segment_v2/ordinal_page_index.cpp | 6 +-
be/src/olap/rowset/segment_v2/ordinal_page_index.h | 7 +-
be/src/olap/rowset/segment_v2/page_decoder.h | 5 +
be/src/olap/rowset/segment_v2/page_io.cpp | 2 +-
be/src/olap/rowset/segment_v2/plain_page.h | 4 +
be/src/olap/rowset/segment_v2/rle_page.h | 23 +
be/src/olap/rowset/segment_v2/segment.cpp | 38 +-
be/src/olap/rowset/segment_v2/segment.h | 9 +-
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 9 +-
be/src/olap/rowset/segment_v2/segment_iterator.h | 3 +
be/src/olap/rowset/segment_v2/zone_map_index.cpp | 2 +-
be/src/olap/rowset/segment_v2/zone_map_index.h | 7 +-
be/src/olap/schema.cpp | 44 +
be/src/olap/schema.h | 4 +
be/src/olap/schema_change.cpp | 11 +-
be/src/olap/segment_loader.cpp | 30 +-
be/src/olap/snapshot_manager.cpp | 70 +-
be/src/olap/snapshot_manager.h | 12 +-
be/src/olap/storage_engine.cpp | 13 +-
be/src/olap/storage_engine.h | 6 +-
be/src/olap/tablet.cpp | 23 +-
be/src/olap/tablet.h | 12 +
be/src/olap/tablet_manager.cpp | 37 +-
be/src/olap/tablet_meta.cpp | 7 +-
be/src/olap/tablet_meta.h | 2 +-
be/src/olap/tablet_schema.cpp | 13 +
be/src/olap/tablet_schema.h | 4 +
be/src/olap/task/engine_clone_task.cpp | 6 +-
be/src/olap/task/engine_storage_migration_task.cpp | 9 +-
be/src/olap/tuple_reader.cpp | 20 +-
be/src/olap/utils.cpp | 116 --
be/src/olap/utils.h | 4 -
be/src/olap/version_graph.cpp | 4 +-
be/src/runtime/CMakeLists.txt | 2 +-
be/src/runtime/datetime_value.cpp | 2 +
be/src/runtime/datetime_value.h | 6 +-
be/src/runtime/descriptors.cpp | 21 +-
be/src/runtime/descriptors.h | 6 +-
be/src/runtime/etl_job_mgr.cpp | 2 +-
be/src/runtime/exec_env.h | 7 +-
be/src/runtime/exec_env_init.cpp | 6 +-
be/src/runtime/export_sink.cpp | 10 +
be/src/runtime/file_result_writer.cpp | 30 +-
be/src/runtime/file_result_writer.h | 9 +-
be/src/runtime/fold_constant_executor.cpp | 267 ++++
be/src/runtime/fold_constant_executor.h | 61 +
be/src/runtime/fold_constant_mgr.cpp | 203 ---
be/src/runtime/fold_constant_mgr.h | 57 -
be/src/runtime/fragment_mgr.cpp | 3 -
be/src/runtime/mysql_result_writer.cpp | 25 +-
be/src/runtime/mysql_result_writer.h | 8 +-
be/src/runtime/plan_fragment_executor.cpp | 95 +-
be/src/runtime/plan_fragment_executor.h | 7 +-
be/src/runtime/primitive_type.h | 77 +-
be/src/runtime/raw_value.h | 56 +
be/src/runtime/result_file_sink.cpp | 4 +-
be/src/runtime/result_sink.cpp | 9 +-
be/src/runtime/result_writer.h | 8 +-
be/src/runtime/row_batch.cpp | 300 ++--
be/src/runtime/row_batch.h | 25 +-
be/src/runtime/runtime_state.cpp | 2 +
be/src/runtime/runtime_state.h | 8 +-
be/src/runtime/snapshot_loader.cpp | 8 +-
be/src/runtime/string_value.hpp | 25 +-
be/src/runtime/tuple.cpp | 64 +-
be/src/runtime/types.h | 58 +
be/src/service/backend_service.cpp | 6 +-
be/src/service/doris_main.cpp | 6 +
be/src/service/internal_service.cpp | 18 +-
be/src/tools/meta_tool.cpp | 8 +-
be/src/udf/udf.cpp | 5 +
be/src/udf/udf.h | 12 +-
be/src/udf/udf_internal.h | 5 +
be/src/udf/udf_ir.cpp | 14 +
be/src/util/CMakeLists.txt | 3 +-
be/src/util/aes_util.cpp | 177 ---
be/src/util/aes_util.h | 37 -
be/src/util/binary_cast.hpp | 17 +-
be/src/util/bitmap_value.h | 137 +-
be/src/util/broker_storage_backend.cpp | 28 +-
be/src/util/broker_storage_backend.h | 9 +-
be/src/util/brpc_stub_cache.h | 2 +-
be/src/util/coding.cpp | 1 -
be/src/util/coding.h | 1 -
be/src/util/encryption_util.cpp | 249 +++
be/src/util/encryption_util.h | 69 +
be/src/util/file_utils.cpp | 106 +-
be/src/util/file_utils.h | 9 +-
be/src/util/path_util.cpp | 7 +
be/src/util/path_util.h | 4 +
be/src/util/proto_util.h | 12 +-
be/src/util/runtime_profile.h | 5 +-
be/src/util/s3_storage_backend.cpp | 112 +-
be/src/util/s3_storage_backend.h | 9 +-
be/src/util/s3_util.cpp | 25 +-
be/src/util/s3_util.h | 10 +
be/src/util/sm3.cpp | 51 +
be/src/util/sm3.h | 44 +
be/src/util/static_asserts.cpp | 3 +-
be/src/util/storage_backend.h | 9 +-
be/src/util/tuple_row_zorder_compare.cpp | 7 +-
be/src/util/url_coding.cpp | 4 +
be/src/vec/CMakeLists.txt | 153 ++
.../vec/aggregate_functions/aggregate_function.h | 237 +++
.../aggregate_functions/aggregate_function_avg.cpp | 68 +
.../aggregate_functions/aggregate_function_avg.h | 128 ++
.../aggregate_function_bitmap.cpp | 87 ++
.../aggregate_function_bitmap.h | 175 +++
.../aggregate_function_combinator.h | 81 +
.../aggregate_function_count.cpp | 52 +
.../aggregate_functions/aggregate_function_count.h | 122 ++
.../aggregate_function_distinct.cpp | 96 ++
.../aggregate_function_distinct.h | 224 +++
.../aggregate_function_hll_union_agg.cpp | 51 +
.../aggregate_function_hll_union_agg.h | 133 ++
.../aggregate_function_min_max.cpp | 85 +
.../aggregate_function_min_max.h | 535 +++++++
.../aggregate_function_nothing.h | 71 +
.../aggregate_function_null.cpp | 92 ++
.../aggregate_functions/aggregate_function_null.h | 291 ++++
.../aggregate_function_reader.cpp | 50 +
.../aggregate_function_reader.h | 35 +
.../aggregate_function_simple_factory.cpp | 62 +
.../aggregate_function_simple_factory.h | 104 ++
.../aggregate_functions/aggregate_function_sum.cpp | 88 ++
.../aggregate_functions/aggregate_function_sum.h | 118 ++
.../aggregate_function_uniq.cpp | 69 +
.../aggregate_functions/aggregate_function_uniq.h | 131 ++
.../aggregate_function_window.cpp | 176 +++
.../aggregate_function_window.h | 434 ++++++
be/src/vec/aggregate_functions/factory_helpers.h | 59 +
be/src/vec/aggregate_functions/helpers.h | 225 +++
.../vec/aggregate_functions/key_holder_helpers.h | 48 +
be/src/vec/columns/collator.cpp | 97 ++
be/src/vec/columns/collator.h | 43 +
be/src/vec/columns/column.cpp | 57 +
be/src/vec/columns/column.h | 483 ++++++
be/src/vec/columns/column_complex.h | 287 ++++
be/src/vec/columns/column_const.cpp | 107 ++
be/src/vec/columns/column_const.h | 174 +++
be/src/vec/columns/column_decimal.cpp | 224 +++
be/src/vec/columns/column_decimal.h | 210 +++
be/src/vec/columns/column_dummy.h | 141 ++
be/src/vec/columns/column_impl.h | 62 +
be/src/vec/columns/column_nothing.h | 46 +
be/src/vec/columns/column_nullable.cpp | 438 ++++++
be/src/vec/columns/column_nullable.h | 238 +++
be/src/vec/columns/column_set.h | 52 +
be/src/vec/columns/column_string.cpp | 375 +++++
be/src/vec/columns/column_string.h | 286 ++++
be/src/vec/columns/column_vector.cpp | 379 +++++
be/src/vec/columns/column_vector.h | 268 ++++
be/src/vec/columns/column_vector_helper.h | 59 +
be/src/vec/columns/columns_common.cpp | 273 ++++
be/src/vec/columns/columns_common.h | 91 ++
be/src/vec/columns/columns_number.h | 45 +
be/src/vec/columns/predicate_column.h | 418 +++++
be/src/vec/common/aggregation_common.h | 269 ++++
be/src/vec/common/allocator.h | 312 ++++
be/src/vec/common/allocator_fwd.h | 30 +
be/src/vec/common/arena.h | 285 ++++
be/src/vec/common/arithmetic_overflow.h | 112 ++
be/src/vec/common/assert_cast.h | 56 +
be/src/vec/common/bit_cast.h | 47 +
be/src/vec/common/bit_helpers.h | 92 ++
be/src/vec/common/columns_hashing.h | 242 +++
be/src/vec/common/columns_hashing_impl.h | 324 ++++
be/src/vec/common/cow.h | 323 ++++
be/src/vec/common/demangle.cpp | 61 +
be/src/vec/common/demangle.h | 34 +
be/src/vec/common/exception.cpp | 202 +++
be/src/vec/common/exception.h | 287 ++++
be/src/vec/common/field_visitors.h | 554 +++++++
be/src/vec/common/hash_table/fixed_hash_map.h | 194 +++
be/src/vec/common/hash_table/fixed_hash_table.h | 383 +++++
be/src/vec/common/hash_table/hash.h | 191 +++
be/src/vec/common/hash_table/hash_map.h | 240 +++
be/src/vec/common/hash_table/hash_set.h | 105 ++
be/src/vec/common/hash_table/hash_table.h | 914 +++++++++++
.../vec/common/hash_table/hash_table_allocator.h | 33 +
.../vec/common/hash_table/hash_table_key_holder.h | 139 ++
be/src/vec/common/int_exp.h | 135 ++
be/src/vec/common/memcmp_small.h | 228 +++
be/src/vec/common/memcpy_small.h | 83 +
be/src/vec/common/mremap.h | 77 +
be/src/vec/common/nan_utils.h | 69 +
be/src/vec/common/pod_array.cpp | 27 +
be/src/vec/common/pod_array.h | 591 +++++++
be/src/vec/common/pod_array_fwd.h | 53 +
be/src/vec/common/radix_sort.h | 389 +++++
be/src/vec/common/sip_hash.h | 231 +++
be/src/vec/common/string_buffer.hpp | 93 ++
be/src/vec/common/string_ref.h | 327 ++++
be/src/vec/common/string_utils/string_utils.cpp | 34 +
be/src/vec/common/string_utils/string_utils.h | 155 ++
be/src/vec/common/strong_typedef.h | 84 +
be/src/vec/common/typeid_cast.h | 62 +
be/src/vec/common/uint128.h | 237 +++
be/src/vec/common/unaligned.h | 42 +
be/src/vec/core/accurate_comparison.h | 567 +++++++
be/src/vec/core/block.cpp | 910 +++++++++++
be/src/vec/core/block.h | 349 +++++
be/src/vec/core/block_info.cpp | 42 +
be/src/vec/core/block_info.h | 76 +
be/src/vec/core/call_on_type_index.h | 260 ++++
be/src/vec/core/column_numbers.h | 30 +
be/src/vec/core/column_with_type_and_name.cpp | 72 +
be/src/vec/core/column_with_type_and_name.h | 56 +
be/src/vec/core/columns_with_type_and_name.h | 31 +
be/src/vec/core/decimal_comparison.h | 303 ++++
be/src/vec/core/field.cpp | 193 +++
be/src/vec/core/field.h | 909 +++++++++++
be/src/vec/core/materialize_block.cpp | 43 +
be/src/vec/core/materialize_block.h | 39 +
be/src/vec/core/names.h | 40 +
be/src/vec/core/sort_block.cpp | 202 +++
be/src/vec/core/sort_block.h | 55 +
be/src/vec/core/sort_cursor.h | 228 +++
be/src/vec/core/sort_description.h | 80 +
be/src/vec/core/types.h | 438 ++++++
be/src/vec/data_types/data_type.cpp | 217 +++
be/src/vec/data_types/data_type.h | 402 +++++
be/src/vec/data_types/data_type_bitmap.cpp | 93 ++
be/src/vec/data_types/data_type_bitmap.h | 86 ++
be/src/vec/data_types/data_type_date.cpp | 66 +
be/src/vec/data_types/data_type_date.h | 42 +
be/src/vec/data_types/data_type_date_time.cpp | 95 ++
be/src/vec/data_types/data_type_date_time.h | 83 +
be/src/vec/data_types/data_type_decimal.cpp | 146 ++
be/src/vec/data_types/data_type_decimal.h | 351 +++++
be/src/vec/data_types/data_type_factory.hpp | 87 ++
be/src/vec/data_types/data_type_nothing.cpp | 42 +
be/src/vec/data_types/data_type_nothing.h | 65 +
be/src/vec/data_types/data_type_nullable.cpp | 113 ++
be/src/vec/data_types/data_type_nullable.h | 90 ++
be/src/vec/data_types/data_type_number.h | 87 ++
be/src/vec/data_types/data_type_number_base.cpp | 123 ++
be/src/vec/data_types/data_type_number_base.h | 66 +
be/src/vec/data_types/data_type_string.cpp | 115 ++
be/src/vec/data_types/data_type_string.h | 57 +
be/src/vec/data_types/get_least_supertype.cpp | 322 ++++
be/src/vec/data_types/get_least_supertype.h | 35 +
be/src/vec/data_types/nested_utils.cpp | 73 +
be/src/vec/data_types/nested_utils.h | 43 +
be/src/vec/data_types/number_traits.h | 275 ++++
be/src/vec/exec/join/join_op.h | 143 ++
be/src/vec/exec/join/vacquire_list.hpp | 54 +
be/src/vec/exec/join/vhash_join_node.cpp | 1063 +++++++++++++
be/src/vec/exec/join/vhash_join_node.h | 225 +++
be/src/vec/exec/vaggregation_node.cpp | 1102 +++++++++++++
be/src/vec/exec/vaggregation_node.h | 483 ++++++
be/src/vec/exec/vanalytic_eval_node.cpp | 658 ++++++++
be/src/vec/exec/vanalytic_eval_node.h | 146 ++
be/src/vec/exec/vassert_num_rows_node.cpp | 99 ++
be/src/vec/exec/vassert_num_rows_node.h | 42 +
be/src/vec/exec/vblocking_join_node.cpp | 140 ++
be/src/vec/exec/vblocking_join_node.h | 132 ++
be/src/vec/exec/vcross_join_node.cpp | 185 +++
be/src/vec/exec/vcross_join_node.h | 85 +
be/src/vec/exec/vempty_set_node.cpp | 30 +
be/src/vec/exec/vempty_set_node.h | 35 +
be/src/vec/exec/ves_http_scan_node.cpp | 176 +++
be/src/vec/exec/ves_http_scan_node.h | 49 +
be/src/vec/exec/ves_http_scanner.cpp | 56 +
be/src/vec/exec/ves_http_scanner.h | 43 +
be/src/vec/exec/vexcept_node.cpp | 112 ++
be/src/vec/exec/vexcept_node.h | 39 +
be/src/vec/exec/vexchange_node.cpp | 104 ++
be/src/vec/exec/vexchange_node.h | 58 +
be/src/vec/exec/vintersect_node.cpp | 111 ++
be/src/vec/exec/vintersect_node.h | 49 +
be/src/vec/exec/vmysql_scan_node.cpp | 124 ++
be/src/vec/exec/vmysql_scan_node.h | 49 +
be/src/vec/exec/vodbc_scan_node.cpp | 130 ++
be/src/vec/exec/vodbc_scan_node.h | 33 +
be/src/vec/exec/volap_scan_node.cpp | 557 +++++++
be/src/vec/exec/volap_scan_node.h | 69 +
be/src/vec/exec/volap_scanner.cpp | 205 +++
be/src/vec/exec/volap_scanner.h | 51 +
be/src/vec/exec/vschema_scan_node.cpp | 263 ++++
be/src/vec/exec/vschema_scan_node.h | 50 +
be/src/vec/exec/vselect_node.cpp | 72 +
be/src/vec/exec/vselect_node.h | 39 +
be/src/vec/exec/vset_operation_node.cpp | 379 +++++
be/src/vec/exec/vset_operation_node.h | 245 +++
be/src/vec/exec/vsort_exec_exprs.cpp | 79 +
be/src/vec/exec/vsort_exec_exprs.h | 98 ++
be/src/vec/exec/vsort_node.cpp | 255 +++
be/src/vec/exec/vsort_node.h | 92 ++
be/src/vec/exec/vunion_node.cpp | 268 ++++
be/src/vec/exec/vunion_node.h | 109 ++
be/src/vec/exprs/vcase_expr.cpp | 114 ++
be/src/vec/exprs/vcase_expr.h | 49 +
be/src/vec/exprs/vcast_expr.cpp | 97 ++
be/src/vec/exprs/vcast_expr.h | 53 +
be/src/vec/exprs/vcompound_pred.h | 42 +
be/src/vec/exprs/vectorized_agg_fn.cpp | 175 +++
be/src/vec/exprs/vectorized_agg_fn.h | 106 ++
be/src/vec/exprs/vectorized_fn_call.cpp | 120 ++
be/src/vec/exprs/vectorized_fn_call.h | 46 +
be/src/vec/exprs/vexpr.cpp | 343 +++++
be/src/vec/exprs/vexpr.h | 168 ++
be/src/vec/exprs/vexpr_context.cpp | 137 ++
be/src/vec/exprs/vexpr_context.h | 91 ++
be/src/vec/exprs/vin_predicate.cpp | 109 ++
be/src/vec/exprs/vin_predicate.h | 52 +
be/src/vec/exprs/vinfo_func.cpp | 59 +
be/src/vec/exprs/vinfo_func.h | 47 +
be/src/vec/exprs/vliteral.cpp | 138 ++
be/src/vec/exprs/vliteral.h | 45 +
be/src/vec/exprs/vslot_ref.cpp | 75 +
be/src/vec/exprs/vslot_ref.h | 50 +
be/src/vec/functions/cast_type_to_either.h | 35 +
be/src/vec/functions/comparison.cpp | 33 +
be/src/vec/functions/comparison_equal_for_null.cpp | 125 ++
be/src/vec/functions/comparison_equals.cpp | 29 +
be/src/vec/functions/comparison_greater.cpp | 29 +
be/src/vec/functions/comparison_less.cpp | 29 +
be/src/vec/functions/date_time_transforms.h | 259 ++++
be/src/vec/functions/divide.cpp | 48 +
be/src/vec/functions/function.cpp | 330 ++++
be/src/vec/functions/function.h | 591 +++++++
.../vec/functions/function_always_not_nullable.h | 85 +
be/src/vec/functions/function_binary_arithmetic.h | 719 +++++++++
.../function_binary_arithmetic_to_null_type.h | 247 +++
be/src/vec/functions/function_bit.cpp | 92 ++
be/src/vec/functions/function_bitmap.cpp | 471 ++++++
be/src/vec/functions/function_case.cpp | 29 +
be/src/vec/functions/function_case.h | 371 +++++
be/src/vec/functions/function_cast.cpp | 27 +
be/src/vec/functions/function_cast.h | 1307 ++++++++++++++++
be/src/vec/functions/function_const.h | 101 ++
.../function_date_or_datetime_computation.cpp | 140 ++
.../function_date_or_datetime_computation.h | 488 ++++++
.../function_date_or_datetime_to_something.h | 93 ++
.../function_date_or_datetime_to_string.cpp | 35 +
.../function_date_or_datetime_to_string.h | 66 +
.../function_datetime_string_to_string.cpp | 32 +
.../functions/function_datetime_string_to_string.h | 99 ++
be/src/vec/functions/function_hash.cpp | 254 +++
be/src/vec/functions/function_hash.h | 40 +
be/src/vec/functions/function_helpers.cpp | 99 ++
be/src/vec/functions/function_helpers.h | 104 ++
be/src/vec/functions/function_ifnull.cpp | 27 +
be/src/vec/functions/function_ifnull.h | 98 ++
be/src/vec/functions/function_json.cpp | 369 +++++
.../vec/functions/function_math_binary_float64.h | 248 +++
be/src/vec/functions/function_math_unary.h | 167 ++
be/src/vec/functions/function_string.cpp | 802 ++++++++++
be/src/vec/functions/function_string.h | 986 ++++++++++++
be/src/vec/functions/function_string_to_string.h | 78 +
be/src/vec/functions/function_timestamp.cpp | 330 ++++
be/src/vec/functions/function_totype.h | 428 ++++++
be/src/vec/functions/function_unary_arithmetic.h | 157 ++
be/src/vec/functions/function_variadic_arguments.h | 79 +
be/src/vec/functions/functions_comparison.h | 475 ++++++
be/src/vec/functions/functions_logical.cpp | 531 +++++++
be/src/vec/functions/functions_logical.h | 167 ++
be/src/vec/functions/hll_cardinality.cpp | 71 +
be/src/vec/functions/hll_empty.cpp | 41 +
be/src/vec/functions/hll_hash.cpp | 100 ++
be/src/vec/functions/if.cpp | 480 ++++++
be/src/vec/functions/in.cpp | 182 +++
be/src/vec/functions/int_div.cpp | 153 ++
be/src/vec/functions/int_div.h | 56 +
be/src/vec/functions/is_not_null.cpp | 76 +
be/src/vec/functions/is_null.cpp | 77 +
be/src/vec/functions/like.cpp | 321 ++++
be/src/vec/functions/like.h | 155 ++
be/src/vec/functions/math.cpp | 481 ++++++
be/src/vec/functions/minus.cpp | 56 +
be/src/vec/functions/modulo.cpp | 159 ++
be/src/vec/functions/multiply.cpp | 56 +
be/src/vec/functions/nullif.cpp | 155 ++
be/src/vec/functions/plus.cpp | 57 +
be/src/vec/functions/random.cpp | 109 ++
be/src/vec/functions/simple_function_factory.h | 175 +++
be/src/vec/functions/time_of_function.cpp | 37 +
be/src/vec/functions/to_time_function.cpp | 54 +
be/src/vec/io/io_helper.h | 405 +++++
be/src/vec/io/reader_buffer.h | 46 +
be/src/vec/io/var_int.h | 176 +++
be/src/vec/olap/block_reader.cpp | 362 +++++
be/src/vec/olap/block_reader.h | 120 ++
be/src/vec/olap/vcollect_iterator.cpp | 380 +++++
be/src/vec/olap/vcollect_iterator.h | 193 +++
be/src/vec/olap/vgeneric_iterators.cpp | 443 ++++++
be/src/vec/olap/vgeneric_iterators.h | 47 +
be/src/vec/runtime/vdata_stream_mgr.cpp | 194 +++
be/src/vec/runtime/vdata_stream_mgr.h | 91 ++
be/src/vec/runtime/vdata_stream_recvr.cpp | 364 +++++
be/src/vec/runtime/vdata_stream_recvr.h | 188 +++
be/src/vec/runtime/vdatetime_value.cpp | 1626 ++++++++++++++++++++
be/src/vec/runtime/vdatetime_value.h | 664 ++++++++
be/src/vec/runtime/vpartition_info.cpp | 56 +
be/src/vec/runtime/vpartition_info.h | 59 +
be/src/vec/runtime/vsorted_run_merger.cpp | 156 ++
be/src/vec/runtime/vsorted_run_merger.h | 91 ++
be/src/vec/sink/mysql_result_writer.cpp | 349 +++++
be/src/vec/sink/mysql_result_writer.h | 70 +
be/src/vec/sink/result_sink.cpp | 137 ++
be/src/vec/sink/result_sink.h | 79 +
be/src/vec/sink/result_writer.h | 29 +
be/src/vec/sink/vdata_stream_sender.cpp | 517 +++++++
be/src/vec/sink/vdata_stream_sender.h | 265 ++++
be/src/vec/sink/vtabet_sink.cpp | 275 ++++
be/src/vec/sink/vtablet_sink.h | 65 +
be/src/vec/utils/util.hpp | 79 +
be/test/env/env_posix_test.cpp | 19 +-
be/test/exec/broker_scanner_test.cpp | 78 +-
be/test/exec/orc_scanner_test.cpp | 15 +-
be/test/exprs/bitmap_function_test.cpp | 270 ++++
be/test/exprs/encryption_functions_test.cpp | 179 ++-
be/test/exprs/math_functions_test.cpp | 31 +-
be/test/exprs/string_functions_test.cpp | 44 +-
be/test/olap/column_reader_test.cpp | 108 ++
be/test/olap/cumulative_compaction_policy_test.cpp | 4 +-
be/test/olap/delete_handler_test.cpp | 12 +-
be/test/olap/file_utils_test.cpp | 8 +-
be/test/olap/fs/file_block_manager_test.cpp | 6 +-
be/test/olap/generic_iterators_test.cpp | 6 +-
be/test/olap/lru_cache_test.cpp | 19 +-
be/test/olap/memory/mem_tablet_test.cpp | 3 +-
be/test/olap/olap_snapshot_converter_test.cpp | 2 +-
be/test/olap/rowset/alpha_rowset_test.cpp | 2 +-
be/test/olap/rowset/beta_rowset_test.cpp | 2 +-
be/test/olap/rowset/rowset_converter_test.cpp | 10 +-
.../rowset/segment_v2/binary_dict_page_test.cpp | 2 +
.../olap/rowset/segment_v2/bitmap_index_test.cpp | 4 +-
.../bloom_filter_index_reader_writer_test.cpp | 4 +-
.../segment_v2/column_reader_writer_test.cpp | 146 +-
.../rowset/segment_v2/ordinal_page_index_test.cpp | 10 +-
be/test/olap/rowset/segment_v2/segment_test.cpp | 20 +-
.../olap/rowset/segment_v2/zone_map_index_test.cpp | 10 +-
be/test/olap/tablet_meta_test.cpp | 2 +-
be/test/olap/tablet_mgr_test.cpp | 8 +-
be/test/olap/tablet_test.cpp | 2 +-
be/test/olap/txn_manager_test.cpp | 10 +-
be/test/plugin/plugin_zip_test.cpp | 4 +-
be/test/tools/benchmark_tool.cpp | 4 +-
be/test/util/CMakeLists.txt | 3 +-
be/test/util/aes_util_test.cpp | 93 --
be/test/util/bitmap_value_test.cpp | 37 +
be/test/util/broker_storage_backend_test.cpp | 2 +-
be/test/util/counts_test.cpp | 2 +-
be/test/util/encryption_util_test.cpp | 145 ++
be/test/util/s3_storage_backend_test.cpp | 4 +-
be/test/util/sm3_test.cpp | 68 +
be/test/util/tuple_row_zorder_compare_test.cpp | 266 ++--
be/test/util/zip_util_test.cpp | 4 +-
be/test/vec/aggregate_functions/CMakeLists.txt | 23 +
be/test/vec/aggregate_functions/agg_test.cpp | 61 +
be/test/vec/core/CMakeLists.txt | 23 +
be/test/vec/core/block_test.cpp | 309 ++++
be/test/vec/core/column_complex_test.cpp | 52 +
be/test/vec/exec/CMakeLists.txt | 21 +
be/test/vec/exec/vgeneric_iterators_test.cpp | 207 +++
be/test/vec/exprs/CMakeLists.txt | 22 +
be/test/vec/exprs/vexpr_test.cpp | 357 +++++
be/test/vec/function/CMakeLists.txt | 31 +
be/test/vec/function/function_abs_test.cpp | 92 ++
be/test/vec/function/function_arithmetic_test.cpp | 142 ++
be/test/vec/function/function_bitmap_test.cpp | 120 ++
be/test/vec/function/function_comparison_test.cpp | 150 ++
be/test/vec/function/function_hash_test.cpp | 103 ++
be/test/vec/function/function_ifnull_test.cpp | 85 +
be/test/vec/function/function_like_test.cpp | 125 ++
be/test/vec/function/function_math_test.cpp | 425 +++++
be/test/vec/function/function_nullif_test.cpp | 75 +
be/test/vec/function/function_string_test.cpp | 690 +++++++++
be/test/vec/function/function_test_util.h | 291 ++++
be/test/vec/function/function_time_test.cpp | 582 +++++++
be/test/vec/runtime/CMakeLists.txt | 22 +
be/test/vec/runtime/vdata_stream_test.cpp | 178 +++
bin/start_be.sh | 50 +-
bin/start_fe.sh | 120 +-
build.sh | 8 +-
contrib/udf/CMakeLists.txt | 8 +-
.../udf/src/udaf_orthogonal_bitmap/CMakeLists.txt | 92 --
.../udf/src/udaf_orthogonal_bitmap/bitmap_value.h | 1327 ----------------
.../orthogonal_bitmap_function.cpp | 492 ------
.../orthogonal_bitmap_function.h | 62 -
.../udf/src/udaf_orthogonal_bitmap/string_value.h | 148 --
docker/Dockerfile | 20 +-
docker/README.md | 4 +
docs/.vuepress/components/CaseList.vue | 102 ++
docs/.vuepress/public/images/Bloom_filter.svg.png | Bin 0 -> 51109 bytes
.../images/binlog/image-20211110145044815.png | Bin 146900 -> 0 bytes
.../images/binlog/image-20211110160106602.png | Bin 52259 -> 0 bytes
.../images/binlog/image-20211110160331479.png | Bin 67624 -> 0 bytes
.../images/binlog/image-20211110160710709.png | Bin 56885 -> 0 bytes
.../public/images/cdc/image-20211025162831632.png | Bin 143949 -> 0 bytes
.../public/images/cdc/image-20211025165547903.png | Bin 39487 -> 0 bytes
.../public/images/cdc/image-20211025170642628.png | Bin 49165 -> 0 bytes
.../public/images/cdc/image-20211025182341086.png | Bin 21677 -> 0 bytes
.../public/images/cdc/image-20211025182435827.png | Bin 48534 -> 0 bytes
.../public/images/cdc/image-20211026095513892.png | Bin 92955 -> 0 bytes
.../public/images/cdc/image-20211026100505972.png | Bin 33173 -> 0 bytes
.../public/images/cdc/image-20211026100804091.png | Bin 10829 -> 0 bytes
.../public/images/cdc/image-20211026100943474.png | Bin 284282 -> 0 bytes
.../public/images/cdc/image-20211026101004547.png | Bin 21854 -> 0 bytes
.../public/images/cdc/image-20211026101203629.png | Bin 43195 -> 0 bytes
.../public/images/image-20210903132250723.png | Bin 250503 -> 0 bytes
.../public/images/image-20210903132539511.png | Bin 54482 -> 0 bytes
.../public/images/image-20210903134043421.png | Bin 333578 -> 0 bytes
docs/.vuepress/sidebar/en.js | 52 +-
docs/.vuepress/sidebar/zh-CN.js | 50 +-
docs/.vuepress/theme/components/Home.vue | 4 +-
docs/.vuepress/theme/layouts/ArticleList.vue | 3 +-
.../en/administrator-guide/block-rule/sql-block.md | 27 +-
docs/en/administrator-guide/bloomfilter.md | 133 ++
docs/en/administrator-guide/config/fe_config.md | 11 +
.../http-actions/compaction-action.md | 2 +
.../load-data/batch-delete-manual.md | 8 +-
.../load-data/broker-load-manual.md | 14 +
.../load-data/routine-load-manual.md | 4 +-
.../operation/be-olap-error-code.md | 6 +-
.../orthogonal-bitmap-manual.md | 159 ++
docs/en/administrator-guide/outfile.md | 161 +-
docs/en/administrator-guide/variables.md | 48 +-
docs/en/article/articles/datax-doris-writer.md | 257 ----
docs/en/article/articles/doris-binlog-load.md | 418 -----
docs/en/article/articles/fe-load-balance.md | 476 ------
docs/en/article/articles/flink-cdc-to-doris.md | 374 -----
docs/en/case-user/case-user.md | 25 +
docs/en/community/commit-format-specification.md | 162 ++
docs/en/developer-guide/How-to-Share-blogs.md | 77 +-
docs/en/developer-guide/bitmap-hll-file-format.md | 64 +
.../developer-guide/commit-format-specification.md | 160 --
docs/en/developer-guide/docker-dev.md | 161 ++
docs/en/developer-guide/fe-idea-dev.md | 85 +-
docs/en/developer-guide/java-format-code.md | 12 +
docs/en/extending-doris/flink-doris-connector.md | 84 +-
docs/en/extending-doris/seatunnel.md | 116 ++
docs/en/extending-doris/spark-doris-connector.md | 4 +-
.../udf/contrib/udaf-orthogonal-bitmap-manual.md | 249 ---
docs/en/faq/faq.md | 50 +-
docs/en/getting-started/data-model-rollup.md | 10 +-
docs/en/installing/compilation.md | 24 +-
.../sql-functions/bitmap-functions/bitmap_and.md | 32 +-
.../bitmap-functions/bitmap_and_count.md | 25 +-
.../sql-functions/bitmap-functions/bitmap_or.md | 32 +-
.../bitmap-functions/bitmap_or_count.md | 16 +-
.../sql-functions/bitmap-functions/bitmap_xor.md | 25 +-
.../bitmap-functions/bitmap_xor_count.md | 25 +-
.../orthogonal_bitmap_intersect.md | 47 +
.../orthogonal_bitmap_intersect_count.md | 46 +
.../orthogonal_bitmap_union_count.md | 46 +
.../sql-functions/encrypt-digest-functions/aes.md | 88 ++
.../sql-functions/encrypt-digest-functions/md5.md | 47 +
.../encrypt-digest-functions/md5sum.md | 56 +
.../sql-functions/encrypt-digest-functions/sm3.md | 47 +
.../encrypt-digest-functions/sm3sum.md | 56 +
.../sql-functions/encrypt-digest-functions/sm4.md | 93 ++
.../table-functions/explode-bitmap.md | 157 ++
.../table-functions/explode-json-array.md | 286 ++++
.../sql-functions/table-functions/explode-split.md | 112 ++
.../Data Manipulation/BROKER LOAD.md | 8 +-
.../sql-statements/Data Manipulation/EXPORT.md | 12 +
.../sql-statements/Data Manipulation/OUTFILE.md | 207 +++
.../Data Manipulation/lateral-view.md | 94 ++
.../administrator-guide/block-rule/sql-block.md | 27 +-
docs/zh-CN/administrator-guide/bloomfilter.md | 133 ++
docs/zh-CN/administrator-guide/config/be_config.md | 1 -
docs/zh-CN/administrator-guide/config/fe_config.md | 11 +
.../http-actions/compaction-action.md | 2 +
.../load-data/batch-delete-manual.md | 10 +-
.../load-data/broker-load-manual.md | 12 +
.../load-data/routine-load-manual.md | 4 +-
.../operation/be-olap-error-code.md | 6 +-
.../orthogonal-bitmap-manual.md | 161 ++
docs/zh-CN/administrator-guide/outfile.md | 161 +-
docs/zh-CN/administrator-guide/variables.md | 44 +-
docs/zh-CN/article/articles/datax-doris-writer.md | 261 ----
docs/zh-CN/article/articles/doris-binlog-load.md | 421 -----
docs/zh-CN/article/articles/fe-load-balance.md | 475 ------
docs/zh-CN/article/articles/flink-cdc-to-doris.md | 372 -----
docs/zh-CN/case-user/case-user.md | 25 +
.../zh-CN/community/commit-format-specification.md | 160 ++
docs/zh-CN/developer-guide/How-to-Share-blogs.md | 75 +-
.../developer-guide/bitmap-hll-file-format.md | 64 +
.../developer-guide/commit-format-specification.md | 158 --
docs/zh-CN/developer-guide/docker-dev.md | 162 ++
docs/zh-CN/developer-guide/fe-idea-dev.md | 27 +-
docs/zh-CN/developer-guide/java-format-code.md | 11 +
docs/zh-CN/downloads/downloads.md | 1 -
.../zh-CN/extending-doris/flink-doris-connector.md | 103 +-
docs/zh-CN/extending-doris/seatunnel.md | 117 ++
.../zh-CN/extending-doris/spark-doris-connector.md | 4 +-
.../udf/contrib/udaf-orthogonal-bitmap-manual.md | 238 ---
docs/zh-CN/faq/faq.md | 54 +-
docs/zh-CN/getting-started/data-model-rollup.md | 12 +-
docs/zh-CN/installing/compilation.md | 24 +-
.../sql-functions/bitmap-functions/bitmap_and.md | 30 +-
.../bitmap-functions/bitmap_and_count.md | 25 +-
.../sql-functions/bitmap-functions/bitmap_or.md | 32 +-
.../bitmap-functions/bitmap_or_count.md | 18 +-
.../sql-functions/bitmap-functions/bitmap_xor.md | 25 +-
.../bitmap-functions/bitmap_xor_count.md | 25 +-
.../orthogonal_bitmap_intersect.md | 47 +
.../orthogonal_bitmap_intersect_count.md | 46 +
.../orthogonal_bitmap_union_count.md | 47 +
.../sql-functions/encrypt-digest-functions/aes.md | 93 ++
.../sql-functions/encrypt-digest-functions/md5.md | 47 +
.../encrypt-digest-functions/md5sum.md | 56 +
.../sql-functions/encrypt-digest-functions/sm3.md | 47 +
.../encrypt-digest-functions/sm3sum.md | 56 +
.../sql-functions/encrypt-digest-functions/sm4.md | 93 ++
.../table-functions/explode-bitmap.md | 157 ++
.../table-functions/explode-json-array.md | 286 ++++
.../sql-functions/table-functions/explode-split.md | 112 ++
.../Data Manipulation/BROKER LOAD.md | 8 +-
.../sql-statements/Data Manipulation/EXPORT.md | 12 +
.../sql-statements/Data Manipulation/OUTFILE.md | 209 +++
.../Data Manipulation/lateral-view.md | 94 ++
env.sh | 35 +-
extension/flink-doris-connector/build.sh | 2 +
extension/flink-doris-connector/pom.xml | 52 +-
.../doris/flink/cfg/DorisExecutionOptions.java | 4 +-
.../flink/datastream/DorisSourceFunction.java | 32 +-
.../org/apache/doris/flink/rest/RestService.java | 6 +-
.../flink/table/DorisDynamicOutputFormat.java | 22 +-
.../doris/flink/table/DorisDynamicTableSource.java | 2 +-
.../apache/doris/flink/table/DorisStreamLoad.java | 36 +-
.../org/apache/doris/flink/DorisSourceExample.java | 2 +-
extension/spark-doris-connector/build.sh | 3 +-
extension/spark-doris-connector/pom.xml | 76 +-
extension/spark-doris-connector/pom_3.0.xml | 76 +-
.../doris/spark/cfg/ConfigurationOptions.java | 6 +-
.../doris/spark/sql/DorisStreamLoadSink.scala | 2 +-
fe/fe-common/pom.xml | 20 +-
.../org/apache/doris/common/io/BitmapValue.java | 425 +++++
.../java/org/apache/doris/common/io/Codec.java | 58 +
.../main/java/org/apache/doris/common/io/Hll.java | 396 +++++
.../org/apache/doris/common/io/Roaring64Map.java | 1467 ++++++++++++++++++
.../apache/doris/common/io/BitmapValueTest.java | 518 +++++++
.../java/org/apache/doris/common/io/HllTest.java | 266 ++++
fe/fe-core/pom.xml | 43 +-
fe/fe-core/src/main/cup/sql_parser.cup | 2 +
.../src/main/java/org/apache/doris/PaloFe.java | 82 +-
.../org/apache/doris/analysis/AnalyticExpr.java | 6 +-
.../java/org/apache/doris/analysis/Analyzer.java | 164 +-
.../org/apache/doris/analysis/ArithmeticExpr.java | 95 +-
.../org/apache/doris/analysis/BinaryPredicate.java | 4 +-
.../org/apache/doris/analysis/DecimalLiteral.java | 9 +
.../apache/doris/analysis/FunctionCallExpr.java | 235 ++-
.../org/apache/doris/analysis/InPredicate.java | 9 +-
.../org/apache/doris/analysis/InlineViewRef.java | 4 +
.../java/org/apache/doris/analysis/InsertStmt.java | 6 +-
.../org/apache/doris/analysis/LateralViewRef.java | 42 +-
.../org/apache/doris/analysis/LikePredicate.java | 5 +-
.../org/apache/doris/analysis/OutFileClause.java | 17 +
.../org/apache/doris/analysis/PredicateUtils.java | 55 +
.../java/org/apache/doris/analysis/QueryStmt.java | 16 +
.../java/org/apache/doris/analysis/SelectStmt.java | 35 +-
.../apache/doris/analysis/SetOperationStmt.java | 12 +-
.../java/org/apache/doris/analysis/SetVar.java | 9 +
.../java/org/apache/doris/analysis/SlotRef.java | 8 +
.../java/org/apache/doris/analysis/TableRef.java | 20 +-
.../java/org/apache/doris/analysis/UpdateStmt.java | 3 +-
.../org/apache/doris/backup/BackupJobInfo.java | 2 +-
.../apache/doris/catalog/AggregateFunction.java | 19 +-
.../java/org/apache/doris/catalog/Catalog.java | 89 +-
.../java/org/apache/doris/catalog/FunctionSet.java | 259 +++-
.../org/apache/doris/catalog/PartitionKey.java | 12 +-
.../apache/doris/catalog/TabletInvertedIndex.java | 5 +-
.../main/java/org/apache/doris/catalog/Type.java | 25 +
.../apache/doris/clone/ClusterLoadStatistic.java | 14 +-
.../org/apache/doris/clone/TabletScheduler.java | 21 +-
.../clone/TwoDimensionalGreedyRebalanceAlgo.java | 11 +
.../main/java/org/apache/doris/common/Config.java | 11 +
.../java/org/apache/doris/common/ErrorCode.java | 43 +-
.../org/apache/doris/common/ThreadPoolManager.java | 35 +-
.../doris/common/proc/FrontendsProcNode.java | 25 +-
.../common/profile/MultiProfileTreeBuilder.java | 2 +-
.../doris/common/profile/PlanTreeBuilder.java | 1 +
.../doris/common/profile/ProfileTreeBuilder.java | 6 +
.../java/org/apache/doris/httpv2/HttpServer.java | 5 +-
.../main/java/org/apache/doris/load/ExportJob.java | 3 +-
.../doris/load/loadv2/BrokerLoadPendingTask.java | 5 +-
.../org/apache/doris/load/loadv2/BulkLoadJob.java | 57 +-
.../java/org/apache/doris/load/loadv2/LoadJob.java | 18 +-
.../apache/doris/load/loadv2/LoadJobScheduler.java | 51 +-
.../apache/doris/load/loadv2/LoadLoadingTask.java | 1 +
.../org/apache/doris/load/loadv2/LoadManager.java | 43 +-
.../doris/load/loadv2/LoadTimeoutChecker.java | 49 -
.../doris/load/loadv2/SparkLoadPendingTask.java | 19 +-
.../load/routineload/KafkaRoutineLoadJob.java | 22 +-
.../doris/load/routineload/RoutineLoadJob.java | 2 +-
.../java/org/apache/doris/mysql/MysqlChannel.java | 4 +-
.../mysql/privilege/CommonUserProperties.java | 3 +-
.../org/apache/doris/planner/AnalyticPlanner.java | 1 +
.../java/org/apache/doris/planner/ColumnBound.java | 66 +
.../java/org/apache/doris/planner/ColumnRange.java | 106 ++
.../apache/doris/planner/DistributedPlanner.java | 1 +
.../org/apache/doris/planner/ExchangeNode.java | 1 +
.../org/apache/doris/planner/HashJoinNode.java | 47 +-
.../doris/planner/ListPartitionPrunerV2.java | 195 +++
.../org/apache/doris/planner/LoadScanNode.java | 3 +-
.../org/apache/doris/planner/OlapScanNode.java | 29 +-
.../org/apache/doris/planner/PartitionPruner.java | 2 +-
.../doris/planner/PartitionPrunerV2Base.java | 200 +++
.../org/apache/doris/planner/PlanFragment.java | 3 +-
.../java/org/apache/doris/planner/PlanNode.java | 11 +-
.../java/org/apache/doris/planner/Planner.java | 1 +
.../doris/planner/RangePartitionPrunerV2.java | 270 ++++
.../java/org/apache/doris/planner/RepeatNode.java | 21 +
.../java/org/apache/doris/planner/ScanNode.java | 198 ++-
.../apache/doris/planner/SingleNodePlanner.java | 5 +-
.../apache/doris/planner/TableFunctionNode.java | 15 +-
.../java/org/apache/doris/qe/ConnectContext.java | 32 +-
.../java/org/apache/doris/qe/ConnectProcessor.java | 47 +-
.../java/org/apache/doris/qe/ConnectScheduler.java | 12 +-
.../main/java/org/apache/doris/qe/Coordinator.java | 3 +-
.../java/org/apache/doris/qe/SessionVariable.java | 49 +-
.../java/org/apache/doris/qe/SimpleScheduler.java | 2 +-
.../java/org/apache/doris/qe/StmtExecutor.java | 69 +-
.../org/apache/doris/qe/cache/CacheAnalyzer.java | 29 +-
.../org/apache/doris/qe/cache/PartitionCache.java | 11 +-
.../java/org/apache/doris/qe/cache/SqlCache.java | 10 +-
.../doris/rewrite/BetweenToCompoundRule.java | 2 +-
.../doris/rewrite/CompoundPredicateWriteRule.java | 2 +-
.../org/apache/doris/rewrite/ExprRewriteRule.java | 2 +-
.../org/apache/doris/rewrite/ExprRewriter.java | 32 +-
.../doris/rewrite/ExtractCommonFactorsRule.java | 4 +-
.../java/org/apache/doris/rewrite/FEFunctions.java | 8 +-
.../apache/doris/rewrite/FoldConstantsRule.java | 6 +-
.../org/apache/doris/rewrite/InferFiltersRule.java | 676 ++++++++
.../rewrite/NormalizeBinaryPredicatesRule.java | 2 +-
.../doris/rewrite/RewriteAliasFunctionRule.java | 2 +-
.../doris/rewrite/RewriteBinaryPredicatesRule.java | 99 ++
.../doris/rewrite/RewriteEncryptKeyRule.java | 2 +-
.../doris/rewrite/RewriteFromUnixTimeRule.java | 2 +-
.../doris/rewrite/RewriteLikePredicateRule.java | 2 +-
...implifyInvalidDateBinaryPredicatesDateRule.java | 2 +-
.../rewrite/mvrewrite/CountDistinctToBitmap.java | 3 +-
.../mvrewrite/CountDistinctToBitmapOrHLLRule.java | 3 +-
.../doris/rewrite/mvrewrite/CountFieldToSum.java | 3 +-
.../rewrite/mvrewrite/HLLHashToSlotRefRule.java | 3 +-
.../apache/doris/rewrite/mvrewrite/NDVToHll.java | 3 +-
.../rewrite/mvrewrite/ToBitmapToSlotRefRule.java | 3 +-
.../apache/doris/service/FrontendServiceImpl.java | 44 +-
.../org/apache/doris/task/MasterTaskExecutor.java | 4 +-
.../doris/transaction/DatabaseTransactionMgr.java | 29 +-
.../apache/doris/transaction/TransactionState.java | 91 +-
.../doris/analysis/ListPartitionPrunerTest.java | 160 +-
.../doris/analysis/PartitionPruneTestBase.java | 71 +
.../doris/analysis/RangePartitionPruneTest.java | 213 +++
.../org/apache/doris/analysis/SelectStmtTest.java | 22 +-
.../analysis/TableNameStoredLowercaseTest.java | 10 +
.../doris/blockrule/SqlBlockRuleMgrTest.java | 29 +
.../java/org/apache/doris/clone/RebalanceTest.java | 4 +-
.../org/apache/doris/load/loadv2/LoadJobTest.java | 14 +-
.../apache/doris/load/loadv2/SparkLoadJobTest.java | 28 +-
.../load/loadv2/SparkLoadPendingTaskTest.java | 11 +-
.../load/routineload/KafkaRoutineLoadJobTest.java | 10 +-
.../org/apache/doris/mysql/MysqlChannelTest.java | 10 +-
.../planner/MaterializedViewFunctionTest.java | 37 +-
.../java/org/apache/doris/planner/PlannerTest.java | 40 +
.../org/apache/doris/planner/QueryPlanTest.java | 26 +-
.../doris/planner/TableFunctionPlanTest.java | 124 +-
.../org/apache/doris/qe/PartitionCacheTest.java | 289 +++-
.../java/org/apache/doris/qe/StmtExecutorTest.java | 2 +-
.../apache/doris/rewrite/InferFiltersRuleTest.java | 309 ++++
.../rewrite/mvrewrite/CountFieldToSumTest.java | 3 +-
fe/pom.xml | 84 +-
fe/spark-dpp/pom.xml | 22 +-
.../main/java/org/apache/doris/common/Codec.java | 58 -
.../apache/doris/load/loadv2/dpp/BitmapValue.java | 425 -----
.../load/loadv2/dpp/DorisKryoRegistrator.java | 3 +
.../java/org/apache/doris/load/loadv2/dpp/Hll.java | 396 -----
.../apache/doris/load/loadv2/dpp/Roaring64Map.java | 1468 ------------------
.../doris/load/loadv2/dpp/SparkRDDAggregator.java | 2 +
.../doris/load/loadv2/dpp/BitmapValueTest.java | 521 -------
.../org/apache/doris/load/loadv2/dpp/HllTest.java | 268 ----
fe_plugins/auditdemo/pom.xml | 10 +-
fe_plugins/auditloader/pom.xml | 15 +-
fe_plugins/pom.xml | 63 +-
fs_brokers/apache_hdfs_broker/bin/start_broker.sh | 71 +-
fs_brokers/apache_hdfs_broker/pom.xml | 58 +-
.../doris/broker/hdfs/FileSystemManager.java | 70 +-
gensrc/proto/data.proto | 1 -
gensrc/proto/internal_service.proto | 4 +
gensrc/proto/olap_file.proto | 7 +
gensrc/proto/palo_internal_service.proto | 46 +
gensrc/script/doris_builtins_functions.py | 318 ++--
gensrc/script/gen_build_version.sh | 59 +-
gensrc/script/gen_builtins_functions.py | 5 +
gensrc/thrift/DataSinks.thrift | 2 -
gensrc/thrift/PaloInternalService.thrift | 5 +
gensrc/thrift/PlanNodes.thrift | 5 +
gensrc/thrift/Types.thrift | 1 +
samples/doris-demo/flink-demo/pom.xml | 5 +-
samples/doris-demo/pom.xml | 2 +-
samples/doris-demo/spark-demo/pom.xml | 4 +-
samples/doris-demo/spring-jdbc-demo/pom.xml | 2 +-
samples/doris-demo/stream-load-demo/pom.xml | 2 +-
thirdparty/CHANGELOG.md | 13 +
thirdparty/build-thirdparty.sh | 15 +
thirdparty/vars.sh | 15 +-
963 files changed, 86868 insertions(+), 13709 deletions(-)
create mode 100644 .asf.yaml
create mode 100644 .github/workflows/build-extension.yml
delete mode 100644 .travis.yml
create mode 100644 be/src/env/env.cpp
create mode 100644 be/src/env/env_posix.h
create mode 100644 be/src/env/env_remote.cpp
create mode 100644 be/src/env/env_remote.h
create mode 100644 be/src/exec/table_function_node.cpp
create mode 100644 be/src/exec/table_function_node.h
create mode 100644 be/src/exprs/table_function/dummy_table_functions.cpp
create mode 100644 be/src/exprs/table_function/dummy_table_functions.h
create mode 100644 be/src/exprs/table_function/explode_bitmap.cpp
create mode 100644 be/src/exprs/table_function/explode_bitmap.h
create mode 100644 be/src/exprs/table_function/explode_json_array.cpp
create mode 100644 be/src/exprs/table_function/explode_json_array.h
create mode 100644 be/src/exprs/table_function/explode_split.cpp
create mode 100644 be/src/exprs/table_function/explode_split.h
create mode 100644 be/src/exprs/table_function/table_function.h
create mode 100644 be/src/exprs/table_function/table_function_factory.cpp
create mode 100644 be/src/exprs/table_function/table_function_factory.h
create mode 100644 be/src/exprs/v_string_functions.h
create mode 100644 be/src/olap/fs/remote_block_manager.cpp
create mode 100644 be/src/olap/fs/remote_block_manager.h
create mode 100644 be/src/runtime/fold_constant_executor.cpp
create mode 100644 be/src/runtime/fold_constant_executor.h
delete mode 100644 be/src/runtime/fold_constant_mgr.cpp
delete mode 100644 be/src/runtime/fold_constant_mgr.h
delete mode 100644 be/src/util/aes_util.cpp
delete mode 100644 be/src/util/aes_util.h
create mode 100644 be/src/util/encryption_util.cpp
create mode 100644 be/src/util/encryption_util.h
create mode 100644 be/src/util/sm3.cpp
create mode 100644 be/src/util/sm3.h
create mode 100644 be/src/vec/CMakeLists.txt
create mode 100644 be/src/vec/aggregate_functions/aggregate_function.h
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_avg.cpp
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_avg.h
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_bitmap.cpp
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_bitmap.h
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_combinator.h
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_count.cpp
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_count.h
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_distinct.cpp
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_distinct.h
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.cpp
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.h
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_min_max.cpp
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_min_max.h
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_nothing.h
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_null.cpp
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_null.h
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_reader.cpp
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_reader.h
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_simple_factory.h
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_sum.cpp
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_sum.h
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_uniq.cpp
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_uniq.h
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_window.cpp
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_window.h
create mode 100644 be/src/vec/aggregate_functions/factory_helpers.h
create mode 100644 be/src/vec/aggregate_functions/helpers.h
create mode 100644 be/src/vec/aggregate_functions/key_holder_helpers.h
create mode 100644 be/src/vec/columns/collator.cpp
create mode 100644 be/src/vec/columns/collator.h
create mode 100644 be/src/vec/columns/column.cpp
create mode 100644 be/src/vec/columns/column.h
create mode 100644 be/src/vec/columns/column_complex.h
create mode 100644 be/src/vec/columns/column_const.cpp
create mode 100644 be/src/vec/columns/column_const.h
create mode 100644 be/src/vec/columns/column_decimal.cpp
create mode 100644 be/src/vec/columns/column_decimal.h
create mode 100644 be/src/vec/columns/column_dummy.h
create mode 100644 be/src/vec/columns/column_impl.h
create mode 100644 be/src/vec/columns/column_nothing.h
create mode 100644 be/src/vec/columns/column_nullable.cpp
create mode 100644 be/src/vec/columns/column_nullable.h
create mode 100644 be/src/vec/columns/column_set.h
create mode 100644 be/src/vec/columns/column_string.cpp
create mode 100644 be/src/vec/columns/column_string.h
create mode 100644 be/src/vec/columns/column_vector.cpp
create mode 100644 be/src/vec/columns/column_vector.h
create mode 100644 be/src/vec/columns/column_vector_helper.h
create mode 100644 be/src/vec/columns/columns_common.cpp
create mode 100644 be/src/vec/columns/columns_common.h
create mode 100644 be/src/vec/columns/columns_number.h
create mode 100644 be/src/vec/columns/predicate_column.h
create mode 100644 be/src/vec/common/aggregation_common.h
create mode 100644 be/src/vec/common/allocator.h
create mode 100644 be/src/vec/common/allocator_fwd.h
create mode 100644 be/src/vec/common/arena.h
create mode 100644 be/src/vec/common/arithmetic_overflow.h
create mode 100644 be/src/vec/common/assert_cast.h
create mode 100644 be/src/vec/common/bit_cast.h
create mode 100644 be/src/vec/common/bit_helpers.h
create mode 100644 be/src/vec/common/columns_hashing.h
create mode 100644 be/src/vec/common/columns_hashing_impl.h
create mode 100644 be/src/vec/common/cow.h
create mode 100644 be/src/vec/common/demangle.cpp
create mode 100644 be/src/vec/common/demangle.h
create mode 100644 be/src/vec/common/exception.cpp
create mode 100644 be/src/vec/common/exception.h
create mode 100644 be/src/vec/common/field_visitors.h
create mode 100644 be/src/vec/common/hash_table/fixed_hash_map.h
create mode 100644 be/src/vec/common/hash_table/fixed_hash_table.h
create mode 100644 be/src/vec/common/hash_table/hash.h
create mode 100644 be/src/vec/common/hash_table/hash_map.h
create mode 100644 be/src/vec/common/hash_table/hash_set.h
create mode 100644 be/src/vec/common/hash_table/hash_table.h
create mode 100644 be/src/vec/common/hash_table/hash_table_allocator.h
create mode 100644 be/src/vec/common/hash_table/hash_table_key_holder.h
create mode 100644 be/src/vec/common/int_exp.h
create mode 100644 be/src/vec/common/memcmp_small.h
create mode 100644 be/src/vec/common/memcpy_small.h
create mode 100644 be/src/vec/common/mremap.h
create mode 100644 be/src/vec/common/nan_utils.h
create mode 100644 be/src/vec/common/pod_array.cpp
create mode 100644 be/src/vec/common/pod_array.h
create mode 100644 be/src/vec/common/pod_array_fwd.h
create mode 100644 be/src/vec/common/radix_sort.h
create mode 100644 be/src/vec/common/sip_hash.h
create mode 100644 be/src/vec/common/string_buffer.hpp
create mode 100644 be/src/vec/common/string_ref.h
create mode 100644 be/src/vec/common/string_utils/string_utils.cpp
create mode 100644 be/src/vec/common/string_utils/string_utils.h
create mode 100644 be/src/vec/common/strong_typedef.h
create mode 100644 be/src/vec/common/typeid_cast.h
create mode 100644 be/src/vec/common/uint128.h
create mode 100644 be/src/vec/common/unaligned.h
create mode 100644 be/src/vec/core/accurate_comparison.h
create mode 100644 be/src/vec/core/block.cpp
create mode 100644 be/src/vec/core/block.h
create mode 100644 be/src/vec/core/block_info.cpp
create mode 100644 be/src/vec/core/block_info.h
create mode 100644 be/src/vec/core/call_on_type_index.h
create mode 100644 be/src/vec/core/column_numbers.h
create mode 100644 be/src/vec/core/column_with_type_and_name.cpp
create mode 100644 be/src/vec/core/column_with_type_and_name.h
create mode 100644 be/src/vec/core/columns_with_type_and_name.h
create mode 100644 be/src/vec/core/decimal_comparison.h
create mode 100644 be/src/vec/core/field.cpp
create mode 100644 be/src/vec/core/field.h
create mode 100644 be/src/vec/core/materialize_block.cpp
create mode 100644 be/src/vec/core/materialize_block.h
create mode 100644 be/src/vec/core/names.h
create mode 100644 be/src/vec/core/sort_block.cpp
create mode 100644 be/src/vec/core/sort_block.h
create mode 100644 be/src/vec/core/sort_cursor.h
create mode 100644 be/src/vec/core/sort_description.h
create mode 100644 be/src/vec/core/types.h
create mode 100644 be/src/vec/data_types/data_type.cpp
create mode 100644 be/src/vec/data_types/data_type.h
create mode 100644 be/src/vec/data_types/data_type_bitmap.cpp
create mode 100644 be/src/vec/data_types/data_type_bitmap.h
create mode 100644 be/src/vec/data_types/data_type_date.cpp
create mode 100644 be/src/vec/data_types/data_type_date.h
create mode 100644 be/src/vec/data_types/data_type_date_time.cpp
create mode 100644 be/src/vec/data_types/data_type_date_time.h
create mode 100644 be/src/vec/data_types/data_type_decimal.cpp
create mode 100644 be/src/vec/data_types/data_type_decimal.h
create mode 100644 be/src/vec/data_types/data_type_factory.hpp
create mode 100644 be/src/vec/data_types/data_type_nothing.cpp
create mode 100644 be/src/vec/data_types/data_type_nothing.h
create mode 100644 be/src/vec/data_types/data_type_nullable.cpp
create mode 100644 be/src/vec/data_types/data_type_nullable.h
create mode 100644 be/src/vec/data_types/data_type_number.h
create mode 100644 be/src/vec/data_types/data_type_number_base.cpp
create mode 100644 be/src/vec/data_types/data_type_number_base.h
create mode 100644 be/src/vec/data_types/data_type_string.cpp
create mode 100644 be/src/vec/data_types/data_type_string.h
create mode 100644 be/src/vec/data_types/get_least_supertype.cpp
create mode 100644 be/src/vec/data_types/get_least_supertype.h
create mode 100644 be/src/vec/data_types/nested_utils.cpp
create mode 100644 be/src/vec/data_types/nested_utils.h
create mode 100644 be/src/vec/data_types/number_traits.h
create mode 100644 be/src/vec/exec/join/join_op.h
create mode 100644 be/src/vec/exec/join/vacquire_list.hpp
create mode 100644 be/src/vec/exec/join/vhash_join_node.cpp
create mode 100644 be/src/vec/exec/join/vhash_join_node.h
create mode 100644 be/src/vec/exec/vaggregation_node.cpp
create mode 100644 be/src/vec/exec/vaggregation_node.h
create mode 100644 be/src/vec/exec/vanalytic_eval_node.cpp
create mode 100644 be/src/vec/exec/vanalytic_eval_node.h
create mode 100644 be/src/vec/exec/vassert_num_rows_node.cpp
create mode 100644 be/src/vec/exec/vassert_num_rows_node.h
create mode 100644 be/src/vec/exec/vblocking_join_node.cpp
create mode 100644 be/src/vec/exec/vblocking_join_node.h
create mode 100644 be/src/vec/exec/vcross_join_node.cpp
create mode 100644 be/src/vec/exec/vcross_join_node.h
create mode 100644 be/src/vec/exec/vempty_set_node.cpp
create mode 100644 be/src/vec/exec/vempty_set_node.h
create mode 100644 be/src/vec/exec/ves_http_scan_node.cpp
create mode 100644 be/src/vec/exec/ves_http_scan_node.h
create mode 100644 be/src/vec/exec/ves_http_scanner.cpp
create mode 100644 be/src/vec/exec/ves_http_scanner.h
create mode 100644 be/src/vec/exec/vexcept_node.cpp
create mode 100644 be/src/vec/exec/vexcept_node.h
create mode 100644 be/src/vec/exec/vexchange_node.cpp
create mode 100644 be/src/vec/exec/vexchange_node.h
create mode 100644 be/src/vec/exec/vintersect_node.cpp
create mode 100644 be/src/vec/exec/vintersect_node.h
create mode 100644 be/src/vec/exec/vmysql_scan_node.cpp
create mode 100644 be/src/vec/exec/vmysql_scan_node.h
create mode 100644 be/src/vec/exec/vodbc_scan_node.cpp
create mode 100644 be/src/vec/exec/vodbc_scan_node.h
create mode 100644 be/src/vec/exec/volap_scan_node.cpp
create mode 100644 be/src/vec/exec/volap_scan_node.h
create mode 100644 be/src/vec/exec/volap_scanner.cpp
create mode 100644 be/src/vec/exec/volap_scanner.h
create mode 100644 be/src/vec/exec/vschema_scan_node.cpp
create mode 100644 be/src/vec/exec/vschema_scan_node.h
create mode 100644 be/src/vec/exec/vselect_node.cpp
create mode 100644 be/src/vec/exec/vselect_node.h
create mode 100644 be/src/vec/exec/vset_operation_node.cpp
create mode 100644 be/src/vec/exec/vset_operation_node.h
create mode 100644 be/src/vec/exec/vsort_exec_exprs.cpp
create mode 100644 be/src/vec/exec/vsort_exec_exprs.h
create mode 100644 be/src/vec/exec/vsort_node.cpp
create mode 100644 be/src/vec/exec/vsort_node.h
create mode 100644 be/src/vec/exec/vunion_node.cpp
create mode 100644 be/src/vec/exec/vunion_node.h
create mode 100644 be/src/vec/exprs/vcase_expr.cpp
create mode 100644 be/src/vec/exprs/vcase_expr.h
create mode 100644 be/src/vec/exprs/vcast_expr.cpp
create mode 100644 be/src/vec/exprs/vcast_expr.h
create mode 100644 be/src/vec/exprs/vcompound_pred.h
create mode 100644 be/src/vec/exprs/vectorized_agg_fn.cpp
create mode 100644 be/src/vec/exprs/vectorized_agg_fn.h
create mode 100644 be/src/vec/exprs/vectorized_fn_call.cpp
create mode 100644 be/src/vec/exprs/vectorized_fn_call.h
create mode 100644 be/src/vec/exprs/vexpr.cpp
create mode 100644 be/src/vec/exprs/vexpr.h
create mode 100644 be/src/vec/exprs/vexpr_context.cpp
create mode 100644 be/src/vec/exprs/vexpr_context.h
create mode 100644 be/src/vec/exprs/vin_predicate.cpp
create mode 100644 be/src/vec/exprs/vin_predicate.h
create mode 100644 be/src/vec/exprs/vinfo_func.cpp
create mode 100644 be/src/vec/exprs/vinfo_func.h
create mode 100644 be/src/vec/exprs/vliteral.cpp
create mode 100644 be/src/vec/exprs/vliteral.h
create mode 100644 be/src/vec/exprs/vslot_ref.cpp
create mode 100644 be/src/vec/exprs/vslot_ref.h
create mode 100644 be/src/vec/functions/cast_type_to_either.h
create mode 100644 be/src/vec/functions/comparison.cpp
create mode 100644 be/src/vec/functions/comparison_equal_for_null.cpp
create mode 100644 be/src/vec/functions/comparison_equals.cpp
create mode 100644 be/src/vec/functions/comparison_greater.cpp
create mode 100644 be/src/vec/functions/comparison_less.cpp
create mode 100644 be/src/vec/functions/date_time_transforms.h
create mode 100644 be/src/vec/functions/divide.cpp
create mode 100644 be/src/vec/functions/function.cpp
create mode 100644 be/src/vec/functions/function.h
create mode 100644 be/src/vec/functions/function_always_not_nullable.h
create mode 100644 be/src/vec/functions/function_binary_arithmetic.h
create mode 100644 be/src/vec/functions/function_binary_arithmetic_to_null_type.h
create mode 100644 be/src/vec/functions/function_bit.cpp
create mode 100644 be/src/vec/functions/function_bitmap.cpp
create mode 100644 be/src/vec/functions/function_case.cpp
create mode 100644 be/src/vec/functions/function_case.h
create mode 100644 be/src/vec/functions/function_cast.cpp
create mode 100644 be/src/vec/functions/function_cast.h
create mode 100644 be/src/vec/functions/function_const.h
create mode 100644 be/src/vec/functions/function_date_or_datetime_computation.cpp
create mode 100644 be/src/vec/functions/function_date_or_datetime_computation.h
create mode 100644 be/src/vec/functions/function_date_or_datetime_to_something.h
create mode 100644 be/src/vec/functions/function_date_or_datetime_to_string.cpp
create mode 100644 be/src/vec/functions/function_date_or_datetime_to_string.h
create mode 100644 be/src/vec/functions/function_datetime_string_to_string.cpp
create mode 100644 be/src/vec/functions/function_datetime_string_to_string.h
create mode 100644 be/src/vec/functions/function_hash.cpp
create mode 100644 be/src/vec/functions/function_hash.h
create mode 100644 be/src/vec/functions/function_helpers.cpp
create mode 100644 be/src/vec/functions/function_helpers.h
create mode 100644 be/src/vec/functions/function_ifnull.cpp
create mode 100644 be/src/vec/functions/function_ifnull.h
create mode 100644 be/src/vec/functions/function_json.cpp
create mode 100644 be/src/vec/functions/function_math_binary_float64.h
create mode 100644 be/src/vec/functions/function_math_unary.h
create mode 100644 be/src/vec/functions/function_string.cpp
create mode 100644 be/src/vec/functions/function_string.h
create mode 100644 be/src/vec/functions/function_string_to_string.h
create mode 100644 be/src/vec/functions/function_timestamp.cpp
create mode 100644 be/src/vec/functions/function_totype.h
create mode 100644 be/src/vec/functions/function_unary_arithmetic.h
create mode 100644 be/src/vec/functions/function_variadic_arguments.h
create mode 100644 be/src/vec/functions/functions_comparison.h
create mode 100644 be/src/vec/functions/functions_logical.cpp
create mode 100644 be/src/vec/functions/functions_logical.h
create mode 100644 be/src/vec/functions/hll_cardinality.cpp
create mode 100644 be/src/vec/functions/hll_empty.cpp
create mode 100644 be/src/vec/functions/hll_hash.cpp
create mode 100644 be/src/vec/functions/if.cpp
create mode 100644 be/src/vec/functions/in.cpp
create mode 100644 be/src/vec/functions/int_div.cpp
create mode 100644 be/src/vec/functions/int_div.h
create mode 100644 be/src/vec/functions/is_not_null.cpp
create mode 100644 be/src/vec/functions/is_null.cpp
create mode 100644 be/src/vec/functions/like.cpp
create mode 100644 be/src/vec/functions/like.h
create mode 100644 be/src/vec/functions/math.cpp
create mode 100644 be/src/vec/functions/minus.cpp
create mode 100644 be/src/vec/functions/modulo.cpp
create mode 100644 be/src/vec/functions/multiply.cpp
create mode 100644 be/src/vec/functions/nullif.cpp
create mode 100644 be/src/vec/functions/plus.cpp
create mode 100644 be/src/vec/functions/random.cpp
create mode 100644 be/src/vec/functions/simple_function_factory.h
create mode 100644 be/src/vec/functions/time_of_function.cpp
create mode 100644 be/src/vec/functions/to_time_function.cpp
create mode 100644 be/src/vec/io/io_helper.h
create mode 100644 be/src/vec/io/reader_buffer.h
create mode 100644 be/src/vec/io/var_int.h
create mode 100644 be/src/vec/olap/block_reader.cpp
create mode 100644 be/src/vec/olap/block_reader.h
create mode 100644 be/src/vec/olap/vcollect_iterator.cpp
create mode 100644 be/src/vec/olap/vcollect_iterator.h
create mode 100644 be/src/vec/olap/vgeneric_iterators.cpp
create mode 100644 be/src/vec/olap/vgeneric_iterators.h
create mode 100644 be/src/vec/runtime/vdata_stream_mgr.cpp
create mode 100644 be/src/vec/runtime/vdata_stream_mgr.h
create mode 100644 be/src/vec/runtime/vdata_stream_recvr.cpp
create mode 100644 be/src/vec/runtime/vdata_stream_recvr.h
create mode 100644 be/src/vec/runtime/vdatetime_value.cpp
create mode 100644 be/src/vec/runtime/vdatetime_value.h
create mode 100644 be/src/vec/runtime/vpartition_info.cpp
create mode 100644 be/src/vec/runtime/vpartition_info.h
create mode 100644 be/src/vec/runtime/vsorted_run_merger.cpp
create mode 100644 be/src/vec/runtime/vsorted_run_merger.h
create mode 100644 be/src/vec/sink/mysql_result_writer.cpp
create mode 100644 be/src/vec/sink/mysql_result_writer.h
create mode 100644 be/src/vec/sink/result_sink.cpp
create mode 100644 be/src/vec/sink/result_sink.h
create mode 100644 be/src/vec/sink/result_writer.h
create mode 100644 be/src/vec/sink/vdata_stream_sender.cpp
create mode 100644 be/src/vec/sink/vdata_stream_sender.h
create mode 100644 be/src/vec/sink/vtabet_sink.cpp
create mode 100644 be/src/vec/sink/vtablet_sink.h
create mode 100644 be/src/vec/utils/util.hpp
delete mode 100644 be/test/util/aes_util_test.cpp
create mode 100644 be/test/util/encryption_util_test.cpp
create mode 100644 be/test/util/sm3_test.cpp
create mode 100644 be/test/vec/aggregate_functions/CMakeLists.txt
create mode 100644 be/test/vec/aggregate_functions/agg_test.cpp
create mode 100644 be/test/vec/core/CMakeLists.txt
create mode 100644 be/test/vec/core/block_test.cpp
create mode 100644 be/test/vec/core/column_complex_test.cpp
create mode 100644 be/test/vec/exec/CMakeLists.txt
create mode 100644 be/test/vec/exec/vgeneric_iterators_test.cpp
create mode 100644 be/test/vec/exprs/CMakeLists.txt
create mode 100644 be/test/vec/exprs/vexpr_test.cpp
create mode 100644 be/test/vec/function/CMakeLists.txt
create mode 100644 be/test/vec/function/function_abs_test.cpp
create mode 100644 be/test/vec/function/function_arithmetic_test.cpp
create mode 100644 be/test/vec/function/function_bitmap_test.cpp
create mode 100644 be/test/vec/function/function_comparison_test.cpp
create mode 100644 be/test/vec/function/function_hash_test.cpp
create mode 100644 be/test/vec/function/function_ifnull_test.cpp
create mode 100644 be/test/vec/function/function_like_test.cpp
create mode 100644 be/test/vec/function/function_math_test.cpp
create mode 100644 be/test/vec/function/function_nullif_test.cpp
create mode 100644 be/test/vec/function/function_string_test.cpp
create mode 100644 be/test/vec/function/function_test_util.h
create mode 100644 be/test/vec/function/function_time_test.cpp
create mode 100644 be/test/vec/runtime/CMakeLists.txt
create mode 100644 be/test/vec/runtime/vdata_stream_test.cpp
delete mode 100644 contrib/udf/src/udaf_orthogonal_bitmap/CMakeLists.txt
delete mode 100644 contrib/udf/src/udaf_orthogonal_bitmap/bitmap_value.h
delete mode 100644 contrib/udf/src/udaf_orthogonal_bitmap/orthogonal_bitmap_function.cpp
delete mode 100644 contrib/udf/src/udaf_orthogonal_bitmap/orthogonal_bitmap_function.h
delete mode 100644 contrib/udf/src/udaf_orthogonal_bitmap/string_value.h
create mode 100644 docs/.vuepress/components/CaseList.vue
create mode 100644 docs/.vuepress/public/images/Bloom_filter.svg.png
delete mode 100644 docs/.vuepress/public/images/binlog/image-20211110145044815.png
delete mode 100644 docs/.vuepress/public/images/binlog/image-20211110160106602.png
delete mode 100644 docs/.vuepress/public/images/binlog/image-20211110160331479.png
delete mode 100644 docs/.vuepress/public/images/binlog/image-20211110160710709.png
delete mode 100644 docs/.vuepress/public/images/cdc/image-20211025162831632.png
delete mode 100644 docs/.vuepress/public/images/cdc/image-20211025165547903.png
delete mode 100644 docs/.vuepress/public/images/cdc/image-20211025170642628.png
delete mode 100644 docs/.vuepress/public/images/cdc/image-20211025182341086.png
delete mode 100644 docs/.vuepress/public/images/cdc/image-20211025182435827.png
delete mode 100644 docs/.vuepress/public/images/cdc/image-20211026095513892.png
delete mode 100644 docs/.vuepress/public/images/cdc/image-20211026100505972.png
delete mode 100644 docs/.vuepress/public/images/cdc/image-20211026100804091.png
delete mode 100644 docs/.vuepress/public/images/cdc/image-20211026100943474.png
delete mode 100644 docs/.vuepress/public/images/cdc/image-20211026101004547.png
delete mode 100644 docs/.vuepress/public/images/cdc/image-20211026101203629.png
delete mode 100644 docs/.vuepress/public/images/image-20210903132250723.png
delete mode 100644 docs/.vuepress/public/images/image-20210903132539511.png
delete mode 100644 docs/.vuepress/public/images/image-20210903134043421.png
create mode 100644 docs/en/administrator-guide/bloomfilter.md
create mode 100644 docs/en/administrator-guide/orthogonal-bitmap-manual.md
delete mode 100644 docs/en/article/articles/datax-doris-writer.md
delete mode 100644 docs/en/article/articles/doris-binlog-load.md
delete mode 100644 docs/en/article/articles/fe-load-balance.md
delete mode 100644 docs/en/article/articles/flink-cdc-to-doris.md
create mode 100644 docs/en/case-user/case-user.md
create mode 100644 docs/en/community/commit-format-specification.md
create mode 100644 docs/en/developer-guide/bitmap-hll-file-format.md
delete mode 100644 docs/en/developer-guide/commit-format-specification.md
create mode 100644 docs/en/developer-guide/docker-dev.md
create mode 100644 docs/en/extending-doris/seatunnel.md
delete mode 100644 docs/en/extending-doris/udf/contrib/udaf-orthogonal-bitmap-manual.md
create mode 100644 docs/en/sql-reference/sql-functions/bitmap-functions/orthogonal_bitmap_intersect.md
create mode 100644 docs/en/sql-reference/sql-functions/bitmap-functions/orthogonal_bitmap_intersect_count.md
create mode 100644 docs/en/sql-reference/sql-functions/bitmap-functions/orthogonal_bitmap_union_count.md
create mode 100644 docs/en/sql-reference/sql-functions/encrypt-digest-functions/aes.md
create mode 100644 docs/en/sql-reference/sql-functions/encrypt-digest-functions/md5.md
create mode 100644 docs/en/sql-reference/sql-functions/encrypt-digest-functions/md5sum.md
create mode 100644 docs/en/sql-reference/sql-functions/encrypt-digest-functions/sm3.md
create mode 100644 docs/en/sql-reference/sql-functions/encrypt-digest-functions/sm3sum.md
create mode 100644 docs/en/sql-reference/sql-functions/encrypt-digest-functions/sm4.md
create mode 100644 docs/en/sql-reference/sql-functions/table-functions/explode-bitmap.md
create mode 100644 docs/en/sql-reference/sql-functions/table-functions/explode-json-array.md
create mode 100644 docs/en/sql-reference/sql-functions/table-functions/explode-split.md
create mode 100644 docs/en/sql-reference/sql-statements/Data Manipulation/OUTFILE.md
create mode 100644 docs/en/sql-reference/sql-statements/Data Manipulation/lateral-view.md
create mode 100644 docs/zh-CN/administrator-guide/bloomfilter.md
create mode 100644 docs/zh-CN/administrator-guide/orthogonal-bitmap-manual.md
delete mode 100644 docs/zh-CN/article/articles/datax-doris-writer.md
delete mode 100644 docs/zh-CN/article/articles/doris-binlog-load.md
delete mode 100644 docs/zh-CN/article/articles/fe-load-balance.md
delete mode 100644 docs/zh-CN/article/articles/flink-cdc-to-doris.md
create mode 100644 docs/zh-CN/case-user/case-user.md
create mode 100644 docs/zh-CN/community/commit-format-specification.md
create mode 100644 docs/zh-CN/developer-guide/bitmap-hll-file-format.md
delete mode 100644 docs/zh-CN/developer-guide/commit-format-specification.md
create mode 100644 docs/zh-CN/developer-guide/docker-dev.md
create mode 100644 docs/zh-CN/extending-doris/seatunnel.md
delete mode 100644 docs/zh-CN/extending-doris/udf/contrib/udaf-orthogonal-bitmap-manual.md
create mode 100644 docs/zh-CN/sql-reference/sql-functions/bitmap-functions/orthogonal_bitmap_intersect.md
create mode 100644 docs/zh-CN/sql-reference/sql-functions/bitmap-functions/orthogonal_bitmap_intersect_count.md
create mode 100644 docs/zh-CN/sql-reference/sql-functions/bitmap-functions/orthogonal_bitmap_union_count.md
create mode 100644 docs/zh-CN/sql-reference/sql-functions/encrypt-digest-functions/aes.md
create mode 100644 docs/zh-CN/sql-reference/sql-functions/encrypt-digest-functions/md5.md
create mode 100644 docs/zh-CN/sql-reference/sql-functions/encrypt-digest-functions/md5sum.md
create mode 100644 docs/zh-CN/sql-reference/sql-functions/encrypt-digest-functions/sm3.md
create mode 100644 docs/zh-CN/sql-reference/sql-functions/encrypt-digest-functions/sm3sum.md
create mode 100644 docs/zh-CN/sql-reference/sql-functions/encrypt-digest-functions/sm4.md
create mode 100644 docs/zh-CN/sql-reference/sql-functions/table-functions/explode-bitmap.md
create mode 100644 docs/zh-CN/sql-reference/sql-functions/table-functions/explode-json-array.md
create mode 100644 docs/zh-CN/sql-reference/sql-functions/table-functions/explode-split.md
create mode 100644 docs/zh-CN/sql-reference/sql-statements/Data Manipulation/OUTFILE.md
create mode 100644 docs/zh-CN/sql-reference/sql-statements/Data Manipulation/lateral-view.md
create mode 100644 fe/fe-common/src/main/java/org/apache/doris/common/io/BitmapValue.java
create mode 100644 fe/fe-common/src/main/java/org/apache/doris/common/io/Codec.java
create mode 100644 fe/fe-common/src/main/java/org/apache/doris/common/io/Hll.java
create mode 100644 fe/fe-common/src/main/java/org/apache/doris/common/io/Roaring64Map.java
create mode 100644 fe/fe-common/src/test/java/org/apache/doris/common/io/BitmapValueTest.java
create mode 100644 fe/fe-common/src/test/java/org/apache/doris/common/io/HllTest.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/PredicateUtils.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadTimeoutChecker.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/planner/ColumnBound.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/planner/ColumnRange.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/planner/ListPartitionPrunerV2.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/planner/PartitionPrunerV2Base.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/planner/RangePartitionPrunerV2.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/rewrite/InferFiltersRule.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/rewrite/RewriteBinaryPredicatesRule.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/analysis/PartitionPruneTestBase.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/analysis/RangePartitionPruneTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/rewrite/InferFiltersRuleTest.java
delete mode 100644 fe/spark-dpp/src/main/java/org/apache/doris/common/Codec.java
delete mode 100644 fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/BitmapValue.java
delete mode 100644 fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/Hll.java
delete mode 100644 fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/Roaring64Map.java
delete mode 100644 fe/spark-dpp/src/test/java/org/apache/doris/load/loadv2/dpp/BitmapValueTest.java
delete mode 100644 fe/spark-dpp/src/test/java/org/apache/doris/load/loadv2/dpp/HllTest.java
create mode 100644 gensrc/proto/palo_internal_service.proto
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 01/01: [Vectorized Exec Engine] Support Vectorized Exec Engine In Doris
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lihaopeng pushed a commit to branch vectorized
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit e0cadc4d6e5a42e2981fb68fe87ce00d436337bb
Author: lihaopeng <li...@baidu.com>
AuthorDate: Fri Dec 31 11:31:31 2021 +0800
[Vectorized Exec Engine] Support Vectorized Exec Engine In Doris
Co-authored-by: HappenLee <ha...@hotmail.com>
Co-authored-by: stdpain <34...@users.noreply.github.com>
Co-authored-by: Zhengguo Yang <ya...@gmail.com>
Co-authored-by: wangbo <50...@qq.com>
Co-authored-by: emmymiao87 <52...@qq.com>
Co-authored-by: Pxl <95...@qq.com>
Co-authored-by: zhangstar333 <87...@users.noreply.github.com>
Co-authored-by: thinker <zc...@qq.com>
Co-authored-by: Zeno Yang <15...@qq.com>
Co-authored-by: Wang Shuo <wa...@gmail.com>
Co-authored-by: zhoubintao <35...@users.noreply.github.com>
Co-authored-by: Gabriel <ga...@gmail.com>
Co-authored-by: xinghuayu007 <14...@qq.com>
Co-authored-by: weizuo93 <we...@apache.org>
Co-authored-by: yiguolei <gu...@tencent.com>
Co-authored-by: anneji-dev <85...@users.noreply.github.com>
Co-authored-by: awakeljw <99...@qq.com>
Co-authored-by: taberylyang <95...@users.noreply.github.com>
Co-authored-by: Cui Kaifeng <48...@users.noreply.github.com>
---
be/CMakeLists.txt | 12 +
be/src/exec/blocking_join_node.cpp | 3 +-
be/src/exec/data_sink.cpp | 15 +-
be/src/exec/data_sink.h | 11 +-
be/src/exec/es/es_scroll_parser.cpp | 381 +++++-
be/src/exec/es/es_scroll_parser.h | 8 +
be/src/exec/es_http_scan_node.cpp | 15 +-
be/src/exec/es_http_scan_node.h | 45 +-
be/src/exec/es_http_scanner.h | 2 +-
be/src/exec/exec_node.cpp | 152 ++-
be/src/exec/exec_node.h | 13 +
be/src/exec/mysql_scan_node.h | 2 +-
be/src/exec/odbc_scan_node.cpp | 12 +-
be/src/exec/odbc_scan_node.h | 12 +-
be/src/exec/olap_scan_node.cpp | 57 +-
be/src/exec/olap_scan_node.h | 8 +-
be/src/exec/olap_scanner.cpp | 7 +-
be/src/exec/partitioned_aggregation_node.cc | 2 +-
be/src/exec/scan_node.h | 2 +-
be/src/exec/schema_scan_node.h | 2 +-
be/src/exec/tablet_info.cpp | 224 ++++
be/src/exec/tablet_info.h | 100 ++
be/src/exec/tablet_sink.cpp | 72 +-
be/src/exec/tablet_sink.h | 6 +-
be/src/exec/text_converter.cpp | 22 +
be/src/exec/text_converter.h | 7 +-
be/src/exec/text_converter.hpp | 133 +-
be/src/exprs/CMakeLists.txt | 1 -
be/src/exprs/aggregate_functions.cpp | 20 +-
be/src/exprs/math_functions.cpp | 6 +-
be/src/exprs/math_functions.h | 2 +-
be/src/exprs/runtime_filter.cpp | 34 +
be/src/exprs/runtime_filter.h | 1 +
be/src/exprs/runtime_filter_slots.h | 38 +-
be/src/exprs/v_string_functions.h | 219 ++++
be/src/olap/block_column_predicate.cpp | 114 ++
be/src/olap/block_column_predicate.h | 26 +
be/src/olap/collect_iterator.cpp | 2 +-
be/src/olap/column_predicate.h | 13 +
be/src/olap/comparison_predicate.cpp | 163 +++
be/src/olap/comparison_predicate.h | 4 +
be/src/olap/generic_iterators.cpp | 62 +-
be/src/olap/generic_iterators.h | 4 +-
be/src/olap/iterators.h | 6 +-
be/src/olap/null_predicate.cpp | 43 +
be/src/olap/null_predicate.h | 8 +
be/src/olap/reader.cpp | 13 +-
be/src/olap/reader.h | 24 +-
be/src/olap/row_block2.cpp | 208 ++++
be/src/olap/row_block2.h | 5 +
be/src/olap/rowset/alpha_rowset_reader.h | 4 +
be/src/olap/rowset/beta_rowset_reader.cpp | 42 +-
be/src/olap/rowset/beta_rowset_reader.h | 1 +
be/src/olap/rowset/rowset_reader.h | 8 +
be/src/olap/rowset/segment_v2/binary_dict_page.cpp | 52 +-
be/src/olap/rowset/segment_v2/binary_dict_page.h | 9 +
be/src/olap/rowset/segment_v2/binary_plain_page.h | 31 +
be/src/olap/rowset/segment_v2/binary_prefix_page.h | 4 +
be/src/olap/rowset/segment_v2/bitshuffle_page.h | 48 +
be/src/olap/rowset/segment_v2/column_reader.cpp | 103 ++
be/src/olap/rowset/segment_v2/column_reader.h | 20 +
.../rowset/segment_v2/empty_segment_iterator.cpp | 6 +-
.../rowset/segment_v2/empty_segment_iterator.h | 3 +-
.../rowset/segment_v2/frame_of_reference_page.h | 4 +
be/src/olap/rowset/segment_v2/page_decoder.h | 5 +
be/src/olap/rowset/segment_v2/plain_page.h | 4 +
be/src/olap/rowset/segment_v2/rle_page.h | 23 +
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 5 +
be/src/olap/rowset/segment_v2/segment_iterator.h | 3 +
be/src/olap/schema.cpp | 44 +
be/src/olap/schema.h | 4 +
be/src/olap/tablet_schema.cpp | 13 +
be/src/olap/tablet_schema.h | 4 +
be/src/olap/tuple_reader.cpp | 2 +-
be/src/runtime/datetime_value.cpp | 2 +
be/src/runtime/datetime_value.h | 6 +-
be/src/runtime/descriptors.cpp | 21 +-
be/src/runtime/descriptors.h | 5 +-
be/src/runtime/exec_env.h | 2 +
be/src/runtime/exec_env_init.cpp | 2 +
be/src/runtime/fold_constant_executor.cpp | 82 +-
be/src/runtime/fold_constant_executor.h | 7 +-
be/src/runtime/mysql_result_writer.cpp | 8 +-
be/src/runtime/mysql_result_writer.h | 2 +
be/src/runtime/plan_fragment_executor.cpp | 93 +-
be/src/runtime/plan_fragment_executor.h | 5 +-
be/src/runtime/primitive_type.h | 77 +-
be/src/runtime/raw_value.h | 56 +
be/src/runtime/result_sink.cpp | 2 +
be/src/runtime/row_batch.cpp | 56 +-
be/src/runtime/row_batch.h | 5 +-
be/src/runtime/string_value.hpp | 25 +-
be/src/runtime/types.h | 58 +
be/src/service/internal_service.cpp | 7 +-
be/src/udf/udf.cpp | 5 +
be/src/udf/udf.h | 5 +
be/src/udf/udf_internal.h | 5 +
be/src/udf/udf_ir.cpp | 14 +
be/src/util/binary_cast.hpp | 17 +-
be/src/util/bitmap_value.h | 6 +
be/src/util/brpc_stub_cache.h | 2 +-
be/src/util/runtime_profile.h | 5 +-
be/src/util/static_asserts.cpp | 3 +-
be/src/util/url_coding.cpp | 4 +
be/src/vec/CMakeLists.txt | 153 +++
.../vec/aggregate_functions/aggregate_function.h | 237 ++++
.../aggregate_functions/aggregate_function_avg.cpp | 68 +
.../aggregate_functions/aggregate_function_avg.h | 128 ++
.../aggregate_function_bitmap.cpp | 87 ++
.../aggregate_function_bitmap.h | 175 +++
.../aggregate_function_combinator.h | 81 ++
.../aggregate_function_count.cpp | 52 +
.../aggregate_functions/aggregate_function_count.h | 122 ++
.../aggregate_function_distinct.cpp | 96 ++
.../aggregate_function_distinct.h | 224 ++++
.../aggregate_function_hll_union_agg.cpp | 51 +
.../aggregate_function_hll_union_agg.h | 133 ++
.../aggregate_function_min_max.cpp | 85 ++
.../aggregate_function_min_max.h | 535 ++++++++
.../aggregate_function_nothing.h | 71 ++
.../aggregate_function_null.cpp | 92 ++
.../aggregate_functions/aggregate_function_null.h | 291 +++++
.../aggregate_function_reader.cpp | 50 +
.../aggregate_function_reader.h} | 30 +-
.../aggregate_function_simple_factory.cpp | 62 +
.../aggregate_function_simple_factory.h | 104 ++
.../aggregate_functions/aggregate_function_sum.cpp | 88 ++
.../aggregate_functions/aggregate_function_sum.h | 118 ++
.../aggregate_function_uniq.cpp | 69 ++
.../aggregate_functions/aggregate_function_uniq.h | 131 ++
.../aggregate_function_window.cpp | 176 +++
.../aggregate_function_window.h | 434 +++++++
be/src/vec/aggregate_functions/factory_helpers.h | 59 +
be/src/vec/aggregate_functions/helpers.h | 225 ++++
.../vec/aggregate_functions/key_holder_helpers.h | 48 +
be/src/vec/columns/collator.cpp | 97 ++
.../columns/collator.h} | 34 +-
be/src/vec/columns/column.cpp | 57 +
be/src/vec/columns/column.h | 483 ++++++++
be/src/vec/columns/column_complex.h | 287 +++++
be/src/vec/columns/column_const.cpp | 107 ++
be/src/vec/columns/column_const.h | 174 +++
be/src/vec/columns/column_decimal.cpp | 224 ++++
be/src/vec/columns/column_decimal.h | 210 ++++
be/src/vec/columns/column_dummy.h | 141 +++
be/src/vec/columns/column_impl.h | 62 +
.../columns/column_nothing.h} | 37 +-
be/src/vec/columns/column_nullable.cpp | 438 +++++++
be/src/vec/columns/column_nullable.h | 238 ++++
be/src/vec/columns/column_set.h | 52 +
be/src/vec/columns/column_string.cpp | 375 ++++++
be/src/vec/columns/column_string.h | 286 +++++
be/src/vec/columns/column_vector.cpp | 379 ++++++
be/src/vec/columns/column_vector.h | 268 ++++
be/src/vec/columns/column_vector_helper.h | 59 +
be/src/vec/columns/columns_common.cpp | 273 ++++
be/src/vec/columns/columns_common.h | 91 ++
.../columns/columns_number.h} | 40 +-
be/src/vec/columns/predicate_column.h | 418 +++++++
be/src/vec/common/aggregation_common.h | 269 ++++
be/src/vec/common/allocator.h | 312 +++++
.../common/allocator_fwd.h} | 26 +-
be/src/vec/common/arena.h | 285 +++++
be/src/vec/common/arithmetic_overflow.h | 112 ++
be/src/vec/common/assert_cast.h | 56 +
be/src/vec/common/bit_cast.h | 47 +
be/src/vec/common/bit_helpers.h | 92 ++
be/src/vec/common/columns_hashing.h | 242 ++++
be/src/vec/common/columns_hashing_impl.h | 324 +++++
be/src/vec/common/cow.h | 323 +++++
be/src/vec/common/demangle.cpp | 61 +
.../common/demangle.h} | 26 +-
be/src/vec/common/exception.cpp | 202 +++
be/src/vec/common/exception.h | 287 +++++
be/src/vec/common/field_visitors.h | 554 +++++++++
be/src/vec/common/hash_table/fixed_hash_map.h | 194 +++
be/src/vec/common/hash_table/fixed_hash_table.h | 383 ++++++
be/src/vec/common/hash_table/hash.h | 191 +++
be/src/vec/common/hash_table/hash_map.h | 240 ++++
be/src/vec/common/hash_table/hash_set.h | 105 ++
be/src/vec/common/hash_table/hash_table.h | 914 ++++++++++++++
.../common/hash_table/hash_table_allocator.h} | 32 +-
.../vec/common/hash_table/hash_table_key_holder.h | 139 +++
be/src/vec/common/int_exp.h | 135 ++
be/src/vec/common/memcmp_small.h | 228 ++++
be/src/vec/common/memcpy_small.h | 83 ++
be/src/vec/common/mremap.h | 77 ++
be/src/vec/common/nan_utils.h | 69 ++
.../common/pod_array.cpp} | 23 +-
be/src/vec/common/pod_array.h | 591 +++++++++
be/src/vec/common/pod_array_fwd.h | 53 +
be/src/vec/common/radix_sort.h | 389 ++++++
be/src/vec/common/sip_hash.h | 231 ++++
be/src/vec/common/string_buffer.hpp | 93 ++
be/src/vec/common/string_ref.h | 327 +++++
.../common/string_utils/string_utils.cpp} | 24 +-
be/src/vec/common/string_utils/string_utils.h | 155 +++
be/src/vec/common/strong_typedef.h | 84 ++
be/src/vec/common/typeid_cast.h | 62 +
be/src/vec/common/uint128.h | 237 ++++
be/src/vec/common/unaligned.h | 42 +
be/src/vec/core/accurate_comparison.h | 567 +++++++++
be/src/vec/core/block.cpp | 910 ++++++++++++++
be/src/vec/core/block.h | 349 ++++++
.../core/block_info.cpp} | 30 +-
be/src/vec/core/block_info.h | 76 ++
be/src/vec/core/call_on_type_index.h | 260 ++++
.../core/column_numbers.h} | 20 +-
be/src/vec/core/column_with_type_and_name.cpp | 72 ++
be/src/vec/core/column_with_type_and_name.h | 56 +
.../core/columns_with_type_and_name.h} | 21 +-
be/src/vec/core/decimal_comparison.h | 303 +++++
be/src/vec/core/field.cpp | 193 +++
be/src/vec/core/field.h | 909 ++++++++++++++
.../core/materialize_block.cpp} | 31 +-
.../core/materialize_block.h} | 36 +-
.../empty_segment_iterator.h => vec/core/names.h} | 35 +-
be/src/vec/core/sort_block.cpp | 202 +++
be/src/vec/core/sort_block.h | 55 +
be/src/vec/core/sort_cursor.h | 228 ++++
be/src/vec/core/sort_description.h | 80 ++
be/src/vec/core/types.h | 438 +++++++
be/src/vec/data_types/data_type.cpp | 217 ++++
be/src/vec/data_types/data_type.h | 402 ++++++
be/src/vec/data_types/data_type_bitmap.cpp | 93 ++
be/src/vec/data_types/data_type_bitmap.h | 86 ++
be/src/vec/data_types/data_type_date.cpp | 66 +
be/src/vec/data_types/data_type_date.h | 42 +
be/src/vec/data_types/data_type_date_time.cpp | 95 ++
be/src/vec/data_types/data_type_date_time.h | 83 ++
be/src/vec/data_types/data_type_decimal.cpp | 146 +++
be/src/vec/data_types/data_type_decimal.h | 351 ++++++
be/src/vec/data_types/data_type_factory.hpp | 87 ++
.../data_types/data_type_nothing.cpp} | 30 +-
be/src/vec/data_types/data_type_nothing.h | 65 +
be/src/vec/data_types/data_type_nullable.cpp | 113 ++
be/src/vec/data_types/data_type_nullable.h | 90 ++
be/src/vec/data_types/data_type_number.h | 87 ++
be/src/vec/data_types/data_type_number_base.cpp | 123 ++
be/src/vec/data_types/data_type_number_base.h | 66 +
be/src/vec/data_types/data_type_string.cpp | 115 ++
be/src/vec/data_types/data_type_string.h | 57 +
be/src/vec/data_types/get_least_supertype.cpp | 322 +++++
.../data_types/get_least_supertype.h} | 27 +-
be/src/vec/data_types/nested_utils.cpp | 73 ++
be/src/vec/data_types/nested_utils.h | 43 +
be/src/vec/data_types/number_traits.h | 275 ++++
be/src/vec/exec/join/join_op.h | 143 +++
be/src/vec/exec/join/vacquire_list.hpp | 54 +
be/src/vec/exec/join/vhash_join_node.cpp | 1063 ++++++++++++++++
be/src/vec/exec/join/vhash_join_node.h | 225 ++++
be/src/vec/exec/vaggregation_node.cpp | 1102 +++++++++++++++++
be/src/vec/exec/vaggregation_node.h | 483 ++++++++
be/src/vec/exec/vanalytic_eval_node.cpp | 658 ++++++++++
be/src/vec/exec/vanalytic_eval_node.h | 146 +++
be/src/vec/exec/vassert_num_rows_node.cpp | 99 ++
.../exec/vassert_num_rows_node.h} | 32 +-
.../exec/vblocking_join_node.cpp} | 104 +-
be/src/vec/exec/vblocking_join_node.h | 132 ++
be/src/vec/exec/vcross_join_node.cpp | 185 +++
be/src/vec/exec/vcross_join_node.h | 85 ++
.../exec/vempty_set_node.cpp} | 24 +-
.../exec/vempty_set_node.h} | 29 +-
be/src/vec/exec/ves_http_scan_node.cpp | 176 +++
.../exec/ves_http_scan_node.h} | 36 +-
be/src/vec/exec/ves_http_scanner.cpp | 56 +
.../exec/ves_http_scanner.h} | 33 +-
be/src/vec/exec/vexcept_node.cpp | 112 ++
.../exec/vexcept_node.h} | 28 +-
be/src/vec/exec/vexchange_node.cpp | 104 ++
be/src/vec/exec/vexchange_node.h | 58 +
be/src/vec/exec/vintersect_node.cpp | 111 ++
be/src/vec/exec/vintersect_node.h | 49 +
be/src/vec/exec/vmysql_scan_node.cpp | 124 ++
.../exec/vmysql_scan_node.h} | 36 +-
be/src/vec/exec/vodbc_scan_node.cpp | 130 ++
.../exec/vodbc_scan_node.h} | 23 +-
be/src/vec/exec/volap_scan_node.cpp | 557 +++++++++
be/src/vec/exec/volap_scan_node.h | 69 ++
be/src/vec/exec/volap_scanner.cpp | 205 +++
be/src/vec/exec/volap_scanner.h | 51 +
be/src/vec/exec/vschema_scan_node.cpp | 263 ++++
.../exec/vschema_scan_node.h} | 37 +-
be/src/vec/exec/vselect_node.cpp | 72 ++
.../exec/vselect_node.h} | 28 +-
be/src/vec/exec/vset_operation_node.cpp | 379 ++++++
be/src/vec/exec/vset_operation_node.h | 245 ++++
be/src/vec/exec/vsort_exec_exprs.cpp | 79 ++
be/src/vec/exec/vsort_exec_exprs.h | 98 ++
be/src/vec/exec/vsort_node.cpp | 255 ++++
be/src/vec/exec/vsort_node.h | 92 ++
be/src/vec/exec/vunion_node.cpp | 268 ++++
be/src/vec/exec/vunion_node.h | 109 ++
be/src/vec/exprs/vcase_expr.cpp | 114 ++
be/src/vec/exprs/vcase_expr.h | 49 +
be/src/vec/exprs/vcast_expr.cpp | 97 ++
be/src/vec/exprs/vcast_expr.h | 53 +
.../exprs/vcompound_pred.h} | 39 +-
be/src/vec/exprs/vectorized_agg_fn.cpp | 175 +++
be/src/vec/exprs/vectorized_agg_fn.h | 106 ++
be/src/vec/exprs/vectorized_fn_call.cpp | 120 ++
be/src/vec/exprs/vectorized_fn_call.h | 46 +
be/src/vec/exprs/vexpr.cpp | 343 +++++
be/src/vec/exprs/vexpr.h | 168 +++
be/src/vec/exprs/vexpr_context.cpp | 137 ++
be/src/vec/exprs/vexpr_context.h | 91 ++
be/src/vec/exprs/vin_predicate.cpp | 109 ++
be/src/vec/exprs/vin_predicate.h | 52 +
be/src/vec/exprs/vinfo_func.cpp | 59 +
.../exprs/vinfo_func.h} | 34 +-
be/src/vec/exprs/vliteral.cpp | 138 +++
.../exprs/vliteral.h} | 32 +-
be/src/vec/exprs/vslot_ref.cpp | 75 ++
be/src/vec/exprs/vslot_ref.h | 50 +
.../functions/cast_type_to_either.h} | 23 +-
.../functions/comparison.cpp} | 27 +-
be/src/vec/functions/comparison_equal_for_null.cpp | 125 ++
.../functions/comparison_equals.cpp} | 23 +-
.../functions/comparison_greater.cpp} | 23 +-
.../functions/comparison_less.cpp} | 23 +-
be/src/vec/functions/date_time_transforms.h | 259 ++++
be/src/vec/functions/divide.cpp | 48 +
be/src/vec/functions/function.cpp | 330 +++++
be/src/vec/functions/function.h | 591 +++++++++
.../vec/functions/function_always_not_nullable.h | 85 ++
be/src/vec/functions/function_binary_arithmetic.h | 719 +++++++++++
.../function_binary_arithmetic_to_null_type.h | 247 ++++
be/src/vec/functions/function_bit.cpp | 92 ++
be/src/vec/functions/function_bitmap.cpp | 471 +++++++
.../functions/function_case.cpp} | 21 +-
be/src/vec/functions/function_case.h | 371 ++++++
.../functions/function_cast.cpp} | 25 +-
be/src/vec/functions/function_cast.h | 1307 ++++++++++++++++++++
be/src/vec/functions/function_const.h | 101 ++
.../function_date_or_datetime_computation.cpp | 140 +++
.../function_date_or_datetime_computation.h | 488 ++++++++
.../function_date_or_datetime_to_something.h | 93 ++
.../function_date_or_datetime_to_string.cpp} | 23 +-
.../function_date_or_datetime_to_string.h | 66 +
.../function_datetime_string_to_string.cpp} | 20 +-
.../functions/function_datetime_string_to_string.h | 99 ++
be/src/vec/functions/function_hash.cpp | 254 ++++
.../functions/function_hash.h} | 35 +-
be/src/vec/functions/function_helpers.cpp | 99 ++
be/src/vec/functions/function_helpers.h | 104 ++
.../functions/function_ifnull.cpp} | 23 +-
be/src/vec/functions/function_ifnull.h | 98 ++
be/src/vec/functions/function_json.cpp | 369 ++++++
.../vec/functions/function_math_binary_float64.h | 248 ++++
be/src/vec/functions/function_math_unary.h | 167 +++
be/src/vec/functions/function_string.cpp | 802 ++++++++++++
be/src/vec/functions/function_string.h | 986 +++++++++++++++
be/src/vec/functions/function_string_to_string.h | 78 ++
be/src/vec/functions/function_timestamp.cpp | 330 +++++
be/src/vec/functions/function_totype.h | 428 +++++++
be/src/vec/functions/function_unary_arithmetic.h | 157 +++
be/src/vec/functions/function_variadic_arguments.h | 79 ++
be/src/vec/functions/functions_comparison.h | 475 +++++++
be/src/vec/functions/functions_logical.cpp | 531 ++++++++
be/src/vec/functions/functions_logical.h | 167 +++
be/src/vec/functions/hll_cardinality.cpp | 71 ++
.../functions/hll_empty.cpp} | 38 +-
be/src/vec/functions/hll_hash.cpp | 100 ++
be/src/vec/functions/if.cpp | 480 +++++++
be/src/vec/functions/in.cpp | 182 +++
be/src/vec/functions/int_div.cpp | 153 +++
be/src/vec/functions/int_div.h | 56 +
be/src/vec/functions/is_not_null.cpp | 76 ++
be/src/vec/functions/is_null.cpp | 77 ++
be/src/vec/functions/like.cpp | 321 +++++
be/src/vec/functions/like.h | 155 +++
be/src/vec/functions/math.cpp | 481 +++++++
be/src/vec/functions/minus.cpp | 56 +
be/src/vec/functions/modulo.cpp | 159 +++
be/src/vec/functions/multiply.cpp | 56 +
be/src/vec/functions/nullif.cpp | 155 +++
be/src/vec/functions/plus.cpp | 57 +
be/src/vec/functions/random.cpp | 109 ++
be/src/vec/functions/simple_function_factory.h | 175 +++
be/src/vec/functions/time_of_function.cpp | 37 +
be/src/vec/functions/to_time_function.cpp | 54 +
be/src/vec/io/io_helper.h | 405 ++++++
.../io/reader_buffer.h} | 36 +-
be/src/vec/io/var_int.h | 176 +++
be/src/vec/olap/block_reader.cpp | 362 ++++++
be/src/vec/olap/block_reader.h | 120 ++
be/src/vec/olap/vcollect_iterator.cpp | 380 ++++++
be/src/vec/olap/vcollect_iterator.h | 193 +++
be/src/vec/olap/vgeneric_iterators.cpp | 443 +++++++
.../olap/vgeneric_iterators.h} | 8 +-
be/src/vec/runtime/vdata_stream_mgr.cpp | 194 +++
be/src/vec/runtime/vdata_stream_mgr.h | 91 ++
be/src/vec/runtime/vdata_stream_recvr.cpp | 364 ++++++
be/src/vec/runtime/vdata_stream_recvr.h | 188 +++
.../runtime/vdatetime_value.cpp} | 247 ++--
.../runtime/vdatetime_value.h} | 161 ++-
be/src/vec/runtime/vpartition_info.cpp | 56 +
be/src/vec/runtime/vpartition_info.h | 59 +
be/src/vec/runtime/vsorted_run_merger.cpp | 156 +++
be/src/vec/runtime/vsorted_run_merger.h | 91 ++
be/src/vec/sink/mysql_result_writer.cpp | 349 ++++++
be/src/{runtime => vec/sink}/mysql_result_writer.h | 48 +-
be/src/{runtime => vec/sink}/result_sink.cpp | 85 +-
be/src/vec/sink/result_sink.h | 79 ++
.../sink/result_writer.h} | 23 +-
be/src/vec/sink/vdata_stream_sender.cpp | 517 ++++++++
be/src/vec/sink/vdata_stream_sender.h | 265 ++++
be/src/vec/sink/vtabet_sink.cpp | 275 ++++
be/src/vec/sink/vtablet_sink.h | 65 +
be/src/vec/utils/util.hpp | 79 ++
be/test/exprs/string_functions_test.cpp | 43 +-
be/test/olap/column_reader_test.cpp | 108 ++
be/test/olap/generic_iterators_test.cpp | 6 +-
.../rowset/segment_v2/binary_dict_page_test.cpp | 2 +
.../segment_v2/column_reader_writer_test.cpp | 122 ++
be/test/tools/benchmark_tool.cpp | 2 +-
be/test/vec/aggregate_functions/CMakeLists.txt | 23 +
be/test/vec/aggregate_functions/agg_test.cpp | 61 +
be/test/vec/core/CMakeLists.txt | 23 +
be/test/vec/core/block_test.cpp | 309 +++++
be/test/vec/core/column_complex_test.cpp | 52 +
be/test/vec/exec/CMakeLists.txt | 21 +
be/test/vec/exec/vgeneric_iterators_test.cpp | 207 ++++
be/test/vec/exprs/CMakeLists.txt | 22 +
be/test/vec/exprs/vexpr_test.cpp | 357 ++++++
be/test/vec/function/CMakeLists.txt | 31 +
be/test/vec/function/function_abs_test.cpp | 92 ++
be/test/vec/function/function_arithmetic_test.cpp | 142 +++
be/test/vec/function/function_bitmap_test.cpp | 120 ++
be/test/vec/function/function_comparison_test.cpp | 150 +++
be/test/vec/function/function_hash_test.cpp | 103 ++
be/test/vec/function/function_ifnull_test.cpp | 85 ++
be/test/vec/function/function_like_test.cpp | 125 ++
be/test/vec/function/function_math_test.cpp | 425 +++++++
be/test/vec/function/function_nullif_test.cpp | 75 ++
be/test/vec/function/function_string_test.cpp | 690 +++++++++++
be/test/vec/function/function_test_util.h | 291 +++++
be/test/vec/function/function_time_test.cpp | 582 +++++++++
be/test/vec/runtime/CMakeLists.txt | 22 +
be/test/vec/runtime/vdata_stream_test.cpp | 178 +++
build.sh | 2 +-
.../org/apache/doris/analysis/AnalyticExpr.java | 6 +-
.../java/org/apache/doris/analysis/Analyzer.java | 25 +
.../org/apache/doris/analysis/ArithmeticExpr.java | 95 +-
.../org/apache/doris/analysis/BinaryPredicate.java | 2 +
.../org/apache/doris/analysis/InPredicate.java | 9 +-
.../org/apache/doris/analysis/LikePredicate.java | 5 +-
.../java/org/apache/doris/analysis/SelectStmt.java | 5 +
.../apache/doris/analysis/SetOperationStmt.java | 5 +-
.../java/org/apache/doris/analysis/TableRef.java | 6 +
.../apache/doris/catalog/AggregateFunction.java | 19 +-
.../java/org/apache/doris/catalog/Catalog.java | 17 +-
.../java/org/apache/doris/catalog/FunctionSet.java | 77 +-
.../main/java/org/apache/doris/catalog/Type.java | 25 +
.../doris/common/profile/PlanTreeBuilder.java | 1 +
.../doris/common/profile/ProfileTreeBuilder.java | 6 +
.../org/apache/doris/planner/AnalyticPlanner.java | 1 +
.../org/apache/doris/planner/ExchangeNode.java | 1 +
.../org/apache/doris/planner/HashJoinNode.java | 47 +-
.../org/apache/doris/planner/OlapScanNode.java | 1 -
.../java/org/apache/doris/planner/PlanNode.java | 9 +-
.../java/org/apache/doris/planner/Planner.java | 1 +
.../apache/doris/planner/SingleNodePlanner.java | 4 +-
.../main/java/org/apache/doris/qe/Coordinator.java | 1 -
.../apache/doris/rewrite/FoldConstantsRule.java | 4 +-
gensrc/proto/palo_internal_service.proto | 46 +
gensrc/script/doris_builtins_functions.py | 255 ++--
gensrc/script/gen_build_version.sh | 2 +-
gensrc/script/gen_builtins_functions.py | 5 +
gensrc/thrift/DataSinks.thrift | 2 -
gensrc/thrift/PlanNodes.thrift | 5 +
471 files changed, 62947 insertions(+), 1397 deletions(-)
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index 2c3f5b7..ae427d1 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -469,6 +469,7 @@ set(DORIS_LINK_LIBS
DorisGen
Webserver
Geo
+ Vec
Plugin
${WL_END_GROUP}
)
@@ -652,6 +653,7 @@ endif()
add_subdirectory(${SRC_DIR}/util)
add_subdirectory(${SRC_DIR}/plugin)
+add_subdirectory(${SRC_DIR}/vec)
# Utility CMake function to make specifying tests and benchmarks less verbose
FUNCTION(ADD_BE_TEST TEST_NAME)
@@ -704,6 +706,11 @@ if (${MAKE_TEST} STREQUAL "ON")
add_subdirectory(${TEST_DIR}/runtime)
add_subdirectory(${TEST_DIR}/udf)
add_subdirectory(${TEST_DIR}/util)
+ add_subdirectory(${TEST_DIR}/vec/core)
+ add_subdirectory(${TEST_DIR}/vec/exprs)
+ add_subdirectory(${TEST_DIR}/vec/function)
+ add_subdirectory(${TEST_DIR}/vec/runtime)
+ add_subdirectory(${TEST_DIR}/vec/aggregate_functions)
add_subdirectory(${TEST_DIR}/plugin)
add_subdirectory(${TEST_DIR}/plugin/example)
add_subdirectory(${TEST_DIR}/tools)
@@ -727,3 +734,8 @@ install(FILES
${BASE_DIR}/../conf/odbcinst.ini
DESTINATION ${OUTPUT_DIR}/conf)
+
+get_property(dirs DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES)
+foreach(dir ${dirs})
+ message(STATUS "dir='${dir}'")
+ endforeach()
diff --git a/be/src/exec/blocking_join_node.cpp b/be/src/exec/blocking_join_node.cpp
index da97701..ba13786 100644
--- a/be/src/exec/blocking_join_node.cpp
+++ b/be/src/exec/blocking_join_node.cpp
@@ -30,7 +30,8 @@ namespace doris {
BlockingJoinNode::BlockingJoinNode(const std::string& node_name, const TJoinOp::type join_op,
ObjectPool* pool, const TPlanNode& tnode,
const DescriptorTbl& descs)
- : ExecNode(pool, tnode, descs), _node_name(node_name), _join_op(join_op) {}
+ : ExecNode(pool, tnode, descs), _node_name(node_name), _join_op(join_op),
+ _left_side_eos(false) {}
Status BlockingJoinNode::init(const TPlanNode& tnode, RuntimeState* state) {
return ExecNode::init(tnode, state);
diff --git a/be/src/exec/data_sink.cpp b/be/src/exec/data_sink.cpp
index aefa087..cf06f09 100644
--- a/be/src/exec/data_sink.cpp
+++ b/be/src/exec/data_sink.cpp
@@ -35,7 +35,10 @@
#include "runtime/result_file_sink.h"
#include "runtime/result_sink.h"
#include "runtime/runtime_state.h"
-#include "util/logging.h"
+
+#include "vec/sink/result_sink.h"
+#include "vec/sink/vdata_stream_sender.h"
+#include "vec/sink/vtablet_sink.h"
namespace doris {
@@ -57,6 +60,9 @@ Status DataSink::create_data_sink(ObjectPool* pool, const TDataSink& thrift_sink
: false;
// TODO: figure out good buffer size based on size of output row
if (is_vec) {
+ tmp_sink = new doris::vectorized::VDataStreamSender(
+ pool, params.sender_id, row_desc, thrift_sink.stream_sink, params.destinations,
+ 16 * 1024, send_query_statistics_with_every_batch);
} else {
tmp_sink = new DataStreamSender(pool, params.sender_id, row_desc,
thrift_sink.stream_sink, params.destinations, 16 * 1024,
@@ -73,6 +79,7 @@ Status DataSink::create_data_sink(ObjectPool* pool, const TDataSink& thrift_sink
// TODO: figure out good buffer size based on size of output row
if (is_vec) {
+ tmp_sink = new doris::vectorized::VResultSink(row_desc, output_exprs, thrift_sink.result_sink, 4096);
} else {
tmp_sink = new ResultSink(row_desc, output_exprs, thrift_sink.result_sink, 1024);
}
@@ -149,7 +156,11 @@ Status DataSink::create_data_sink(ObjectPool* pool, const TDataSink& thrift_sink
case TDataSinkType::OLAP_TABLE_SINK: {
Status status;
DCHECK(thrift_sink.__isset.olap_table_sink);
- sink->reset(new stream_load::OlapTableSink(pool, row_desc, output_exprs, &status));
+ if (is_vec) {
+ sink->reset(new stream_load::VOlapTableSink(pool, row_desc, output_exprs, &status));
+ } else {
+ sink->reset(new stream_load::OlapTableSink(pool, row_desc, output_exprs, &status));
+ }
RETURN_IF_ERROR(status);
break;
}
diff --git a/be/src/exec/data_sink.h b/be/src/exec/data_sink.h
index 30f5580..fcec10a 100644
--- a/be/src/exec/data_sink.h
+++ b/be/src/exec/data_sink.h
@@ -33,11 +33,13 @@ class ObjectPool;
class RowBatch;
class RuntimeProfile;
class RuntimeState;
-class TPlanExecRequest;
-class TPlanExecParams;
class TPlanFragmentExecParams;
class RowDescriptor;
+namespace vectorized {
+class Block;
+}
+
// Superclass of all data sinks.
class DataSink {
public:
@@ -56,8 +58,11 @@ public:
// Send a row batch into this sink.
// eos should be true when the last batch is passed to send()
virtual Status send(RuntimeState* state, RowBatch* batch) = 0;
- // virtual Status send(RuntimeState* state, RowBatch* batch, bool eos) = 0;
+ // Send a Block into this sink.
+ virtual Status send(RuntimeState* state, vectorized::Block* block) {
+ return Status::NotSupported("Not support send block");
+ };
// Releases all resources that were allocated in prepare()/send().
// Further send() calls are illegal after calling close().
// It must be okay to call this multiple times. Subsequent calls should
diff --git a/be/src/exec/es/es_scroll_parser.cpp b/be/src/exec/es/es_scroll_parser.cpp
index 396c086..86cd16a 100644
--- a/be/src/exec/es/es_scroll_parser.cpp
+++ b/be/src/exec/es/es_scroll_parser.cpp
@@ -31,6 +31,7 @@
#include "runtime/mem_pool.h"
#include "runtime/mem_tracker.h"
#include "util/string_parser.hpp"
+#include "vec/runtime/vdatetime_value.h"
namespace doris {
@@ -79,6 +80,8 @@ static const std::string ERROR_MEM_LIMIT_EXCEEDED =
static const std::string ERROR_COL_DATA_IS_ARRAY =
"Data source returned an array for the type $0"
"based on column metadata.";
+static const std::string INVALID_NULL_VALUE =
+ "Invalid null value occurs: Non-null column `$0` contains NULL";
#define RETURN_ERROR_IF_COL_IS_ARRAY(col, type) \
do { \
@@ -169,7 +172,7 @@ static Status get_int_value(const rapidjson::Value& col, PrimitiveType type, voi
template <typename T>
static Status get_float_value(const rapidjson::Value& col, PrimitiveType type, void* slot,
bool pure_doc_value) {
- DCHECK(sizeof(T) == 4 || sizeof(T) == 8);
+ static_assert(sizeof(T) == 4 || sizeof(T) == 8);
if (col.IsNumber()) {
*reinterpret_cast<T*>(slot) = (T)(sizeof(T) == 4 ? col.GetFloat() : col.GetDouble());
return Status::OK();
@@ -193,6 +196,68 @@ static Status get_float_value(const rapidjson::Value& col, PrimitiveType type, v
return Status::OK();
}
+template <typename T>
+static Status insert_float_value(const rapidjson::Value& col, PrimitiveType type,
+ vectorized::IColumn* col_ptr, bool pure_doc_value,
+ bool nullable) {
+ static_assert(sizeof(T) == 4 || sizeof(T) == 8);
+ if (col.IsNumber() && nullable) {
+ T value = (T)(sizeof(T) == 4 ? col.GetFloat() : col.GetDouble());
+ col_ptr->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&value)), 0);
+ return Status::OK();
+ }
+
+ if (pure_doc_value && col.IsArray() && nullable) {
+ T value = (T)(sizeof(T) == 4 ? col[0].GetFloat() : col[0].GetDouble());
+ col_ptr->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&value)), 0);
+ return Status::OK();
+ }
+
+ RETURN_ERROR_IF_COL_IS_ARRAY(col, type);
+ RETURN_ERROR_IF_COL_IS_NOT_STRING(col, type);
+
+ StringParser::ParseResult result;
+ const std::string& val = col.GetString();
+ size_t len = col.GetStringLength();
+ T v = StringParser::string_to_float<T>(val.c_str(), len, &result);
+ RETURN_ERROR_IF_PARSING_FAILED(result, col, type);
+
+ col_ptr->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&v)), 0);
+
+ return Status::OK();
+}
+
+template <typename T>
+static Status insert_int_value(const rapidjson::Value& col, PrimitiveType type,
+ vectorized::IColumn* col_ptr, bool pure_doc_value,
+ bool nullable) {
+ if (col.IsNumber()) {
+ T value = (T)(sizeof(T) < 8 ? col.GetInt() : col.GetInt64());
+ col_ptr->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&value)), 0);
+ return Status::OK();
+ }
+
+ if (pure_doc_value && col.IsArray()) {
+ RETURN_ERROR_IF_COL_IS_NOT_NUMBER(col[0], type);
+ T value = (T)(sizeof(T) < 8 ? col[0].GetInt() : col[0].GetInt64());
+ col_ptr->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&value)), 0);
+ return Status::OK();
+ }
+
+ RETURN_ERROR_IF_COL_IS_ARRAY(col, type);
+ RETURN_ERROR_IF_COL_IS_NOT_STRING(col, type);
+
+ StringParser::ParseResult result;
+ const std::string& val = col.GetString();
+ size_t len = col.GetStringLength();
+ T v = StringParser::string_to_int<T>(val.c_str(), len, &result);
+ RETURN_ERROR_IF_PARSING_FAILED(result, col, type);
+
+ col_ptr->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&v)), 0);
+
+ return Status::OK();
+}
+
ScrollParser::ScrollParser(bool doc_value_mode)
: _scroll_id(""), _size(0), _line_index(0), _doc_value_mode(doc_value_mode) {}
@@ -426,22 +491,53 @@ Status ScrollParser::fill_tuple(const TupleDescriptor* tuple_desc, Tuple* tuple,
*reinterpret_cast<int8_t*>(slot) = col.GetInt();
break;
}
- if (pure_doc_value && col.IsArray()) {
+
+ bool is_nested_str = false;
+ if (pure_doc_value && col.IsArray() && col[0].IsBool()) {
*reinterpret_cast<int8_t*>(slot) = col[0].GetBool();
break;
+ } else if (pure_doc_value && col.IsArray() && col[0].IsString()) {
+ is_nested_str = true;
+ } else if (pure_doc_value && col.IsArray()) {
+ return Status::InternalError(
+ strings::Substitute(ERROR_INVALID_COL_DATA, "BOOLEAN"));
}
- RETURN_ERROR_IF_COL_IS_ARRAY(col, type);
- RETURN_ERROR_IF_COL_IS_NOT_STRING(col, type);
-
- const std::string& val = col.GetString();
- size_t val_size = col.GetStringLength();
+ const rapidjson::Value& str_col = is_nested_str? col[0]: col;
+ const std::string& val = str_col.GetString();
+ size_t val_size = str_col.GetStringLength();
StringParser::ParseResult result;
bool b = StringParser::string_to_bool(val.c_str(), val_size, &result);
- RETURN_ERROR_IF_PARSING_FAILED(result, col, type);
+ RETURN_ERROR_IF_PARSING_FAILED(result, str_col, type);
*reinterpret_cast<int8_t*>(slot) = b;
break;
}
+ case TYPE_DECIMALV2: {
+ DecimalV2Value data;
+
+ if (col.IsDouble()) {
+ data.assign_from_double(col.GetDouble());
+ } else {
+ std::string val;
+ if (pure_doc_value) {
+ if (!col[0].IsString()) {
+ val = json_value_to_string(col[0]);
+ } else {
+ val = col[0].GetString();
+ }
+ } else {
+ RETURN_ERROR_IF_COL_IS_ARRAY(col, type);
+ if (!col.IsString()) {
+ val = json_value_to_string(col);
+ } else {
+ val = col.GetString();
+ }
+ }
+ data.parse_from_str(val.data(), val.length());
+ }
+ reinterpret_cast<DecimalV2Value*>(slot)->set_value(data.value());
+ break;
+ }
case TYPE_DATE:
case TYPE_DATETIME: {
@@ -482,6 +578,235 @@ Status ScrollParser::fill_tuple(const TupleDescriptor* tuple_desc, Tuple* tuple,
return Status::OK();
}
+Status ScrollParser::fill_columns(const TupleDescriptor* tuple_desc,
+ std::vector<vectorized::MutableColumnPtr>& columns,
+ MemPool* tuple_pool, bool* line_eof,
+ const std::map<std::string, std::string>& docvalue_context) {
+ *line_eof = true;
+
+ if (_size <= 0 || _line_index >= _size) {
+ return Status::OK();
+ }
+
+ const rapidjson::Value& obj = _inner_hits_node[_line_index++];
+ bool pure_doc_value = false;
+ if (obj.HasMember("fields")) {
+ pure_doc_value = true;
+ }
+ const rapidjson::Value& line = obj.HasMember(FIELD_SOURCE) ? obj[FIELD_SOURCE] : obj["fields"];
+
+ for (int i = 0; i < tuple_desc->slots().size(); ++i) {
+ const SlotDescriptor* slot_desc = tuple_desc->slots()[i];
+ auto col_ptr = columns[i].get();
+
+ if (!slot_desc->is_materialized()) {
+ continue;
+ }
+ if (slot_desc->col_name() == FIELD_ID) {
+ // actually this branch will not be reached, this is guaranteed by Doris FE.
+ if (pure_doc_value) {
+ std::stringstream ss;
+ ss << "obtain `_id` is not supported in doc_values mode";
+ return Status::RuntimeError(ss.str());
+ }
+ // obj[FIELD_ID] must not be NULL
+ std::string _id = obj[FIELD_ID].GetString();
+ size_t len = _id.length();
+
+ col_ptr->insert_data(const_cast<const char*>(_id.data()), len);
+ continue;
+ }
+
+ const char* col_name = pure_doc_value ? docvalue_context.at(slot_desc->col_name()).c_str()
+ : slot_desc->col_name().c_str();
+
+ rapidjson::Value::ConstMemberIterator itr = line.FindMember(col_name);
+ if (itr == line.MemberEnd() && slot_desc->is_nullable()) {
+ auto nullable_column = reinterpret_cast<vectorized::ColumnNullable*>(col_ptr);
+ nullable_column->insert_data(nullptr, 0);
+ continue;
+ } else if (itr == line.MemberEnd() && !slot_desc->is_nullable()) {
+ std::string details = strings::Substitute(INVALID_NULL_VALUE, col_name);
+ return Status::RuntimeError(details);
+ }
+
+ const rapidjson::Value& col = line[col_name];
+
+ PrimitiveType type = slot_desc->type().type;
+
+ // when the column value is null, the subsequent type casting will report an error
+ if (col.IsNull() && slot_desc->is_nullable()) {
+ col_ptr->insert_data(nullptr, 0);
+ continue;
+ } else if (col.IsNull() && !slot_desc->is_nullable()) {
+ std::string details = strings::Substitute(INVALID_NULL_VALUE, col_name);
+ return Status::RuntimeError(details);
+ }
+ switch (type) {
+ case TYPE_CHAR:
+ case TYPE_VARCHAR:
+ case TYPE_STRING: {
+ // sometimes elasticsearch user post some not-string value to Elasticsearch Index.
+ // because of reading value from _source, we can not process all json type and then just transfer the value to original string representation
+ // this may be a tricky, but we can workaround this issue
+ std::string val;
+ if (pure_doc_value) {
+ if (!col[0].IsString()) {
+ val = json_value_to_string(col[0]);
+ } else {
+ val = col[0].GetString();
+ }
+ } else {
+ RETURN_ERROR_IF_COL_IS_ARRAY(col, type);
+ if (!col.IsString()) {
+ val = json_value_to_string(col);
+ } else {
+ val = col.GetString();
+ }
+ }
+ size_t val_size = val.length();
+ col_ptr->insert_data(
+ const_cast<const char*>(val.data()), val_size);
+ break;
+ }
+
+ case TYPE_TINYINT: {
+ insert_int_value<int8_t>(col, type, col_ptr, pure_doc_value, slot_desc->is_nullable());
+ break;
+ }
+
+ case TYPE_SMALLINT: {
+ insert_int_value<int16_t>(col, type, col_ptr, pure_doc_value, slot_desc->is_nullable());
+ break;
+ }
+
+ case TYPE_INT: {
+ insert_int_value<int32>(col, type, col_ptr, pure_doc_value, slot_desc->is_nullable());
+ break;
+ }
+
+ case TYPE_BIGINT: {
+ insert_int_value<int64_t>(col, type, col_ptr, pure_doc_value, slot_desc->is_nullable());
+ break;
+ }
+
+ case TYPE_LARGEINT: {
+ insert_int_value<__int128>(col, type, col_ptr, pure_doc_value, slot_desc->is_nullable());
+ break;
+ }
+
+ case TYPE_DOUBLE: {
+ insert_float_value<double>(col, type, col_ptr, pure_doc_value, slot_desc->is_nullable());
+ break;
+ }
+
+ case TYPE_FLOAT: {
+ insert_float_value<float>(col, type, col_ptr, pure_doc_value, slot_desc->is_nullable());
+ break;
+ }
+
+ case TYPE_BOOLEAN: {
+ if (col.IsBool()) {
+ int8_t val = col.GetBool();
+ col_ptr->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&val)), 0);
+ break;
+ }
+
+ if (col.IsNumber()) {
+ int8_t val = col.GetInt();
+ col_ptr->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&val)), 0);
+ break;
+ }
+
+ bool is_nested_str = false;
+ if (pure_doc_value && col.IsArray() && col[0].IsBool()) {
+ int8_t val = col[0].GetBool();
+ col_ptr->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&val)), 0);
+ break;
+ } else if (pure_doc_value && col.IsArray() && col[0].IsString()) {
+ is_nested_str = true;
+ } else if (pure_doc_value && col.IsArray()) {
+ return Status::InternalError(
+ strings::Substitute(ERROR_INVALID_COL_DATA, "BOOLEAN"));
+ }
+
+ const rapidjson::Value& str_col = is_nested_str? col[0]: col;
+
+ const std::string& val = str_col.GetString();
+ size_t val_size = str_col.GetStringLength();
+ StringParser::ParseResult result;
+ bool b = StringParser::string_to_bool(val.c_str(), val_size, &result);
+ RETURN_ERROR_IF_PARSING_FAILED(result, str_col, type);
+ col_ptr->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&b)), 0);
+ break;
+ }
+ case TYPE_DECIMALV2: {
+ DecimalV2Value data;
+
+ if (col.IsDouble()) {
+ data.assign_from_double(col.GetDouble());
+ } else {
+ std::string val;
+ if (pure_doc_value) {
+ if (!col[0].IsString()) {
+ val = json_value_to_string(col[0]);
+ } else {
+ val = col[0].GetString();
+ }
+ } else {
+ RETURN_ERROR_IF_COL_IS_ARRAY(col, type);
+ if (!col.IsString()) {
+ val = json_value_to_string(col);
+ } else {
+ val = col.GetString();
+ }
+ }
+ data.parse_from_str(val.data(), val.length());
+ }
+ col_ptr->insert_data(
+ const_cast<const char*>(reinterpret_cast<char*>(&data)), 0);
+ break;
+ }
+
+ case TYPE_DATE:
+ case TYPE_DATETIME: {
+ // this would happend just only when `enable_docvalue_scan = false`, and field has timestamp format date from _source
+ if (col.IsNumber()) {
+ // ES process date/datetime field would use millisecond timestamp for index or docvalue
+ // processing date type field, if a number is encountered, Doris On ES will force it to be processed according to ms
+ // Doris On ES needs to be consistent with ES, so just divided by 1000 because the unit for from_unixtime is seconds
+ RETURN_IF_ERROR(fill_date_col_with_timestamp(col_ptr, col, type));
+ } else if (col.IsArray() && pure_doc_value) {
+ // this would happened just only when `enable_docvalue_scan = true`
+ // ES add default format for all field after ES 6.4, if we not provided format for `date` field ES would impose
+ // a standard date-format for date field as `2020-06-16T00:00:00.000Z`
+ // At present, we just process this string format date. After some PR were merged into Doris, we would impose `epoch_mills` for
+ // date field's docvalue
+ if (col[0].IsString()) {
+ RETURN_IF_ERROR(fill_date_col_with_strval(col_ptr, col[0], type));
+ break;
+ }
+ // ES would return millisecond timestamp for date field, divided by 1000 because the unit for from_unixtime is seconds
+ RETURN_IF_ERROR(fill_date_col_with_timestamp(col_ptr, col, type));
+ } else {
+ // this would happened just only when `enable_docvalue_scan = false`, and field has string format date from _source
+ RETURN_ERROR_IF_COL_IS_ARRAY(col, type);
+ RETURN_ERROR_IF_COL_IS_NOT_STRING(col, type);
+ RETURN_IF_ERROR(fill_date_col_with_strval(col_ptr, col, type));
+ }
+ break;
+ }
+ default: {
+ DCHECK(false);
+ break;
+ }
+ }
+ }
+
+ *line_eof = false;
+ return Status::OK();
+}
+
Status ScrollParser::fill_date_slot_with_strval(void* slot, const rapidjson::Value& col,
PrimitiveType type) {
DateTimeValue* ts_slot = reinterpret_cast<DateTimeValue*>(slot);
@@ -511,4 +836,44 @@ Status ScrollParser::fill_date_slot_with_timestamp(void* slot, const rapidjson::
return Status::OK();
}
+Status ScrollParser::fill_date_col_with_strval(vectorized::IColumn* col_ptr,
+ const rapidjson::Value& col, PrimitiveType type) {
+ vectorized::VecDateTimeValue dt_val;
+ const std::string& val = col.GetString();
+ size_t val_size = col.GetStringLength();
+ if (!dt_val.from_date_str(val.c_str(), val_size)) {
+ RETURN_ERROR_IF_CAST_FORMAT_ERROR(col, type);
+ }
+ if (type == TYPE_DATE) {
+ dt_val.cast_to_date();
+ } else {
+ dt_val.to_datetime();
+ }
+
+ auto date_packed_int = binary_cast<doris::vectorized::VecDateTimeValue, int64_t>(
+ *reinterpret_cast<vectorized::VecDateTimeValue*>(&dt_val));
+ col_ptr->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&date_packed_int)), 0);
+ return Status::OK();
+}
+
+Status ScrollParser::fill_date_col_with_timestamp(vectorized::IColumn* col_ptr,
+ const rapidjson::Value& col, PrimitiveType type) {
+ vectorized::VecDateTimeValue dt_val;
+ if (!dt_val.from_unixtime(col.GetInt64() / 1000, "+08:00")) {
+ RETURN_ERROR_IF_CAST_FORMAT_ERROR(col, type);
+ }
+ if (type == TYPE_DATE) {
+ reinterpret_cast<vectorized::VecDateTimeValue*>(&dt_val)->cast_to_date();
+ } else {
+ reinterpret_cast<vectorized::VecDateTimeValue*>(&dt_val)->set_type(TIME_DATETIME);
+ }
+
+ auto date_packed_int = binary_cast<doris::vectorized::VecDateTimeValue, int64_t>(
+ *reinterpret_cast<vectorized::VecDateTimeValue*>(&dt_val));
+ col_ptr->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&date_packed_int)), 0);
+
+ return Status::OK();
+}
+
+
} // namespace doris
diff --git a/be/src/exec/es/es_scroll_parser.h b/be/src/exec/es/es_scroll_parser.h
index f2ff855..30eabaf 100644
--- a/be/src/exec/es/es_scroll_parser.h
+++ b/be/src/exec/es/es_scroll_parser.h
@@ -22,6 +22,7 @@
#include "rapidjson/document.h"
#include "runtime/descriptors.h"
#include "runtime/tuple.h"
+#include "vec/core/block.h"
namespace doris {
@@ -35,6 +36,9 @@ public:
Status parse(const std::string& scroll_result, bool exactly_once = false);
Status fill_tuple(const TupleDescriptor* _tuple_desc, Tuple* tuple, MemPool* mem_pool,
bool* line_eof, const std::map<std::string, std::string>& docvalue_context);
+ Status fill_columns(const TupleDescriptor* _tuple_desc,
+ std::vector<vectorized::MutableColumnPtr>& columns, MemPool* mem_pool,
+ bool* line_eof, const std::map<std::string, std::string>& docvalue_context);
const std::string& get_scroll_id();
int get_size();
@@ -44,9 +48,13 @@ private:
// type is used for distinguish date and datetime
// fill date slot with string format date
Status fill_date_slot_with_strval(void* slot, const rapidjson::Value& col, PrimitiveType type);
+ Status fill_date_col_with_strval(vectorized::IColumn* col_ptr, const rapidjson::Value& col,
+ PrimitiveType type);
// fill date slot with timestamp
Status fill_date_slot_with_timestamp(void* slot, const rapidjson::Value& col,
PrimitiveType type);
+ Status fill_date_col_with_timestamp(vectorized::IColumn* col_ptr, const rapidjson::Value& col,
+ PrimitiveType type);
private:
std::string _scroll_id;
diff --git a/be/src/exec/es_http_scan_node.cpp b/be/src/exec/es_http_scan_node.cpp
index 012110d..7b67486 100644
--- a/be/src/exec/es_http_scan_node.cpp
+++ b/be/src/exec/es_http_scan_node.cpp
@@ -437,10 +437,17 @@ void EsHttpScanNode::scanner_worker(int start_idx, int length, std::promise<Stat
properties, _column_names, _predicates, _docvalue_context, &doc_value_mode);
// start scanner to scan
- std::unique_ptr<EsHttpScanner> scanner(
- new EsHttpScanner(_runtime_state, runtime_profile(), _tuple_id, properties,
- scanner_expr_ctxs, &counter, doc_value_mode));
- status = scanner_scan(std::move(scanner), scanner_expr_ctxs, &counter);
+ if (!_vectorized) {
+ std::unique_ptr<EsHttpScanner> scanner(
+ new EsHttpScanner(_runtime_state, runtime_profile(), _tuple_id, properties,
+ scanner_expr_ctxs, &counter, doc_value_mode));
+ status = scanner_scan(std::move(scanner), scanner_expr_ctxs, &counter);
+ } else {
+ std::unique_ptr<VEsHttpScanner> scanner(
+ new VEsHttpScanner(_runtime_state, runtime_profile(), _tuple_id, properties,
+ scanner_expr_ctxs, &counter, doc_value_mode));
+ status = scanner_scan(std::move(scanner));
+ }
if (!status.ok()) {
LOG(WARNING) << "Scanner[" << start_idx
<< "] process failed. status=" << status.get_error_msg();
diff --git a/be/src/exec/es_http_scan_node.h b/be/src/exec/es_http_scan_node.h
index 640ea0c..b90bf0e 100644
--- a/be/src/exec/es_http_scan_node.h
+++ b/be/src/exec/es_http_scan_node.h
@@ -29,6 +29,7 @@
#include "common/status.h"
#include "exec/es_http_scanner.h"
+#include "vec/exec/ves_http_scanner.h"
#include "exec/scan_node.h"
#include "gen_cpp/PaloInternalService_types.h"
@@ -56,7 +57,6 @@ protected:
// Write debug string of this into out.
virtual void debug_string(int indentation_level, std::stringstream* out) const override;
-private:
// Update process status to one failed status,
// NOTE: Must hold the mutex of this scan node
bool update_status(const Status& new_status) {
@@ -67,20 +67,8 @@ private:
return false;
}
- // Create scanners to do scan job
- Status start_scanners();
-
- // Collect all scanners 's status
- Status collect_scanners_status();
-
// One scanner worker, This scanner will handle 'length' ranges start from start_idx
- void scanner_worker(int start_idx, int length, std::promise<Status>& p_status);
-
- // Scan one range
- Status scanner_scan(std::unique_ptr<EsHttpScanner> scanner,
- const std::vector<ExprContext*>& conjunct_ctxs, EsScanCounter* counter);
-
- Status build_conjuncts_list();
+ virtual void scanner_worker(int start_idx, int length, std::promise<Status>& p_status);
TupleId _tuple_id;
RuntimeState* _runtime_state;
@@ -92,20 +80,41 @@ private:
int _max_buffered_batches;
RuntimeProfile::Counter* _wait_scanner_timer;
- bool _all_scanners_finished;
Status _process_status;
+ std::map<std::string, std::string> _docvalue_context;
+
+ std::condition_variable _queue_reader_cond;
+ std::condition_variable _queue_writer_cond;
+ bool _vectorized = false;
+
+private:
+ // Create scanners to do scan job
+ Status start_scanners();
+
+ // Collect all scanners 's status
+ Status collect_scanners_status();
+
+ // Scan one range
+ Status scanner_scan(std::unique_ptr<EsHttpScanner> scanner,
+ const std::vector<ExprContext*>& conjunct_ctxs, EsScanCounter* counter);
+
+ virtual Status scanner_scan(std::unique_ptr<VEsHttpScanner> scanner) {
+ return Status::NotSupported("vectorized scan in EsHttpScanNode is not supported!");
+ };
+
+ Status build_conjuncts_list();
+
+ bool _all_scanners_finished;
+
std::vector<std::thread> _scanner_threads;
std::vector<std::promise<Status>> _scanners_status;
std::map<std::string, std::string> _properties;
- std::map<std::string, std::string> _docvalue_context;
std::map<std::string, std::string> _fields_context;
std::vector<TScanRangeParams> _scan_ranges;
std::vector<std::string> _column_names;
std::mutex _batch_queue_lock;
- std::condition_variable _queue_reader_cond;
- std::condition_variable _queue_writer_cond;
std::deque<std::shared_ptr<RowBatch>> _batch_queue;
std::vector<EsPredicate*> _predicates;
diff --git a/be/src/exec/es_http_scanner.h b/be/src/exec/es_http_scanner.h
index 2bade5a..dcebfe1 100644
--- a/be/src/exec/es_http_scanner.h
+++ b/be/src/exec/es_http_scanner.h
@@ -68,7 +68,7 @@ public:
void close();
-private:
+protected:
RuntimeState* _state;
RuntimeProfile* _profile;
TupleId _tuple_id;
diff --git a/be/src/exec/exec_node.cpp b/be/src/exec/exec_node.cpp
index d692a79..b18160e 100644
--- a/be/src/exec/exec_node.cpp
+++ b/be/src/exec/exec_node.cpp
@@ -63,6 +63,25 @@
#include "util/debug_util.h"
#include "util/runtime_profile.h"
+#include "vec/core/block.h"
+#include "vec/exec/join/vhash_join_node.h"
+#include "vec/exec/vaggregation_node.h"
+#include "vec/exec/ves_http_scan_node.h"
+#include "vec/exec/vcross_join_node.h"
+#include "vec/exec/vexchange_node.h"
+#include "vec/exec/vmysql_scan_node.h"
+#include "vec/exec/vodbc_scan_node.h"
+#include "vec/exec/volap_scan_node.h"
+#include "vec/exec/vsort_node.h"
+#include "vec/exec/vunion_node.h"
+#include "vec/exec/vintersect_node.h"
+#include "vec/exec/vexcept_node.h"
+#include "vec/exec/vanalytic_eval_node.h"
+#include "vec/exec/vassert_num_rows_node.h"
+#include "vec/exec/vselect_node.h"
+#include "vec/exprs/vexpr.h"
+#include "vec/exec/vempty_set_node.h"
+#include "vec/exec/vschema_scan_node.h"
namespace doris {
const std::string ExecNode::ROW_THROUGHPUT_COUNTER = "RowsReturnedRate";
@@ -167,6 +186,9 @@ Status ExecNode::init(const TPlanNode& tnode, RuntimeState* state) {
init_runtime_profile(profile);
if (tnode.__isset.vconjunct) {
+ _vconjunct_ctx_ptr.reset(new doris::vectorized::VExprContext*);
+ RETURN_IF_ERROR(doris::vectorized::VExpr::create_expr_tree(_pool, tnode.vconjunct,
+ _vconjunct_ctx_ptr.get()));
}
RETURN_IF_ERROR(Expr::create_expr_trees(_pool, tnode.conjuncts, &_conjunct_ctxs));
@@ -189,6 +211,9 @@ Status ExecNode::prepare(RuntimeState* state) {
_mem_tracker);
_expr_mem_pool.reset(new MemPool(_expr_mem_tracker.get()));
+ if (_vconjunct_ctx_ptr) {
+ RETURN_IF_ERROR((*_vconjunct_ctx_ptr)->prepare(state, row_desc(), expr_mem_tracker()));
+ }
RETURN_IF_ERROR(Expr::prepare(_conjunct_ctxs, state, row_desc(), expr_mem_tracker()));
// TODO(zc):
@@ -202,6 +227,9 @@ Status ExecNode::prepare(RuntimeState* state) {
Status ExecNode::open(RuntimeState* state) {
RETURN_IF_ERROR(exec_debug_action(TExecNodePhase::OPEN));
+ if (_vconjunct_ctx_ptr) {
+ RETURN_IF_ERROR((*_vconjunct_ctx_ptr)->open(state));
+ }
return Expr::open(_conjunct_ctxs, state);
}
@@ -240,6 +268,7 @@ Status ExecNode::close(RuntimeState* state) {
}
}
+ if (_vconjunct_ctx_ptr) (*_vconjunct_ctx_ptr)->close(state);
Expr::close(_conjunct_ctxs, state);
if (expr_mem_pool() != nullptr) {
@@ -339,12 +368,41 @@ Status ExecNode::create_tree_helper(RuntimeState* state, ObjectPool* pool,
Status ExecNode::create_node(RuntimeState* state, ObjectPool* pool, const TPlanNode& tnode,
const DescriptorTbl& descs, ExecNode** node) {
+ std::stringstream error_msg;
+
if (state->enable_vectorized_exec()) {
- return Status::InternalError("unsupport enable_vectorized_engine");
+ switch (tnode.node_type) {
+ case TPlanNodeType::OLAP_SCAN_NODE:
+ case TPlanNodeType::ASSERT_NUM_ROWS_NODE:
+ case TPlanNodeType::HASH_JOIN_NODE:
+ case TPlanNodeType::AGGREGATION_NODE:
+ case TPlanNodeType::UNION_NODE:
+ case TPlanNodeType::CROSS_JOIN_NODE:
+ case TPlanNodeType::SORT_NODE:
+ case TPlanNodeType::EXCHANGE_NODE:
+ case TPlanNodeType::ODBC_SCAN_NODE:
+ case TPlanNodeType::MYSQL_SCAN_NODE:
+ case TPlanNodeType::INTERSECT_NODE:
+ case TPlanNodeType::EXCEPT_NODE:
+ case TPlanNodeType::ES_HTTP_SCAN_NODE:
+ case TPlanNodeType::EMPTY_SET_NODE:
+ case TPlanNodeType::SCHEMA_SCAN_NODE:
+ case TPlanNodeType::ANALYTIC_EVAL_NODE:
+ case TPlanNodeType::SELECT_NODE:
+ break;
+ default: {
+ const auto& i = _TPlanNodeType_VALUES_TO_NAMES.find(tnode.node_type);
+ const char* str = "unknown node type";
+
+ if (i != _TPlanNodeType_VALUES_TO_NAMES.end()) {
+ str = i->second;
+ }
+ error_msg << "V" << str << " not implemented";
+ return Status::InternalError(error_msg.str());
+ }
+ }
}
- std::stringstream error_msg;
-
VLOG_CRITICAL << "tnode:\n" << apache::thrift::ThriftDebugString(tnode);
switch (tnode.node_type) {
case TPlanNodeType::CSV_SCAN_NODE:
@@ -353,14 +411,20 @@ Status ExecNode::create_node(RuntimeState* state, ObjectPool* pool, const TPlanN
case TPlanNodeType::MYSQL_SCAN_NODE:
#ifdef DORIS_WITH_MYSQL
- *node = pool->add(new MysqlScanNode(pool, tnode, descs));
+ if (state->enable_vectorized_exec()) {
+ *node = pool->add(new vectorized::VMysqlScanNode(pool, tnode, descs));
+ } else
+ *node = pool->add(new MysqlScanNode(pool, tnode, descs));
return Status::OK();
#else
return Status::InternalError(
"Don't support MySQL table, you should rebuild Doris with WITH_MYSQL option ON");
#endif
case TPlanNodeType::ODBC_SCAN_NODE:
- *node = pool->add(new OdbcScanNode(pool, tnode, descs));
+ if (state->enable_vectorized_exec()) {
+ *node = pool->add(new vectorized::VOdbcScanNode(pool, tnode, descs));
+ } else
+ *node = pool->add(new OdbcScanNode(pool, tnode, descs));
return Status::OK();
case TPlanNodeType::ES_SCAN_NODE:
@@ -368,15 +432,24 @@ Status ExecNode::create_node(RuntimeState* state, ObjectPool* pool, const TPlanN
return Status::OK();
case TPlanNodeType::ES_HTTP_SCAN_NODE:
- *node = pool->add(new EsHttpScanNode(pool, tnode, descs));
+ if (state->enable_vectorized_exec()) {
+ *node = pool->add(new vectorized::VEsHttpScanNode(pool, tnode, descs));
+ } else {
+ *node = pool->add(new EsHttpScanNode(pool, tnode, descs));
+ }
return Status::OK();
case TPlanNodeType::SCHEMA_SCAN_NODE:
- *node = pool->add(new SchemaScanNode(pool, tnode, descs));
+ if (state->enable_vectorized_exec()) {
+ *node = pool->add(new vectorized::VSchemaScanNode(pool, tnode, descs));
+ } else {
+ *node = pool->add(new SchemaScanNode(pool, tnode, descs));
+ }
return Status::OK();
case TPlanNodeType::OLAP_SCAN_NODE:
if (state->enable_vectorized_exec()) {
+ *node = pool->add(new vectorized::VOlapScanNode(pool, tnode, descs));
} else {
*node = pool->add(new OlapScanNode(pool, tnode, descs));
}
@@ -384,6 +457,7 @@ Status ExecNode::create_node(RuntimeState* state, ObjectPool* pool, const TPlanN
case TPlanNodeType::AGGREGATION_NODE:
if (state->enable_vectorized_exec()) {
+ *node = pool->add(new vectorized::AggregationNode(pool, tnode, descs));
} else {
if (config::enable_partitioned_aggregation) {
*node = pool->add(new PartitionedAggregationNode(pool, tnode, descs));
@@ -394,11 +468,16 @@ Status ExecNode::create_node(RuntimeState* state, ObjectPool* pool, const TPlanN
return Status::OK();
case TPlanNodeType::HASH_JOIN_NODE:
- *node = pool->add(new HashJoinNode(pool, tnode, descs));
+ if (state->enable_vectorized_exec()) {
+ *node = pool->add(new vectorized::HashJoinNode(pool, tnode, descs));
+ } else {
+ *node = pool->add(new HashJoinNode(pool, tnode, descs));
+ }
return Status::OK();
case TPlanNodeType::CROSS_JOIN_NODE:
if (state->enable_vectorized_exec()) {
+ *node = pool->add(new vectorized::VCrossJoinNode(pool, tnode, descs));
} else {
*node = pool->add(new CrossJoinNode(pool, tnode, descs));
}
@@ -409,18 +488,27 @@ Status ExecNode::create_node(RuntimeState* state, ObjectPool* pool, const TPlanN
return Status::OK();
case TPlanNodeType::EMPTY_SET_NODE:
- *node = pool->add(new EmptySetNode(pool, tnode, descs));
+ if (state->enable_vectorized_exec()) {
+ *node = pool->add(new vectorized::VEmptySetNode(pool, tnode, descs));
+ } else {
+ *node = pool->add(new EmptySetNode(pool, tnode, descs));
+ }
return Status::OK();
case TPlanNodeType::EXCHANGE_NODE:
if (state->enable_vectorized_exec()) {
+ *node = pool->add(new doris::vectorized::VExchangeNode(pool, tnode, descs));
} else {
*node = pool->add(new ExchangeNode(pool, tnode, descs));
}
return Status::OK();
case TPlanNodeType::SELECT_NODE:
- *node = pool->add(new SelectNode(pool, tnode, descs));
+ if (state->enable_vectorized_exec()) {
+ *node = pool->add(new doris::vectorized::VSelectNode(pool, tnode, descs));
+ } else {
+ *node = pool->add(new SelectNode(pool, tnode, descs));
+ }
return Status::OK();
case TPlanNodeType::OLAP_REWRITE_NODE:
@@ -429,6 +517,7 @@ Status ExecNode::create_node(RuntimeState* state, ObjectPool* pool, const TPlanN
case TPlanNodeType::SORT_NODE:
if (state->enable_vectorized_exec()) {
+ *node = pool->add(new vectorized::VSortNode(pool, tnode, descs));
} else {
if (tnode.sort_node.use_top_n) {
*node = pool->add(new TopNNode(pool, tnode, descs));
@@ -439,8 +528,12 @@ Status ExecNode::create_node(RuntimeState* state, ObjectPool* pool, const TPlanN
return Status::OK();
case TPlanNodeType::ANALYTIC_EVAL_NODE:
- *node = pool->add(new AnalyticEvalNode(pool, tnode, descs));
- break;
+ if (state->enable_vectorized_exec()) {
+ *node = pool->add(new vectorized::VAnalyticEvalNode(pool, tnode, descs));
+ } else {
+ *node = pool->add(new AnalyticEvalNode(pool, tnode, descs));
+ }
+ return Status::OK();
case TPlanNodeType::MERGE_NODE:
*node = pool->add(new MergeNode(pool, tnode, descs));
@@ -448,17 +541,26 @@ Status ExecNode::create_node(RuntimeState* state, ObjectPool* pool, const TPlanN
case TPlanNodeType::UNION_NODE:
if (state->enable_vectorized_exec()) {
+ *node = pool->add(new vectorized::VUnionNode(pool, tnode, descs));
} else {
*node = pool->add(new UnionNode(pool, tnode, descs));
}
return Status::OK();
case TPlanNodeType::INTERSECT_NODE:
- *node = pool->add(new IntersectNode(pool, tnode, descs));
+ if (state->enable_vectorized_exec()) {
+ *node = pool->add(new vectorized::VIntersectNode(pool, tnode, descs));
+ } else {
+ *node = pool->add(new IntersectNode(pool, tnode, descs));
+ }
return Status::OK();
case TPlanNodeType::EXCEPT_NODE:
- *node = pool->add(new ExceptNode(pool, tnode, descs));
+ if (state->enable_vectorized_exec()) {
+ *node = pool->add(new vectorized::VExceptNode(pool, tnode, descs));
+ } else {
+ *node = pool->add(new ExceptNode(pool, tnode, descs));
+ }
return Status::OK();
case TPlanNodeType::BROKER_SCAN_NODE:
@@ -470,7 +572,11 @@ Status ExecNode::create_node(RuntimeState* state, ObjectPool* pool, const TPlanN
return Status::OK();
case TPlanNodeType::ASSERT_NUM_ROWS_NODE:
- *node = pool->add(new AssertNumRowsNode(pool, tnode, descs));
+ if (state->enable_vectorized_exec()) {
+ *node = pool->add(new vectorized::VAssertNumRowsNode(pool, tnode, descs));
+ } else {
+ *node = pool->add(new AssertNumRowsNode(pool, tnode, descs));
+ }
return Status::OK();
case TPlanNodeType::TABLE_FUNCTION_NODE:
@@ -637,6 +743,22 @@ Status ExecNode::claim_buffer_reservation(RuntimeState* state) {
Status ExecNode::release_unused_reservation() {
return _buffer_pool_client.DecreaseReservationTo(_resource_profile.min_reservation);
}
+
+void ExecNode::release_block_memory(vectorized::Block& block, uint16_t child_idx) {
+ DCHECK(child_idx < _children.size());
+ block.clear_column_data(child(child_idx)->row_desc().num_materialized_slots());
+}
+
+void ExecNode::reached_limit(vectorized::Block* block, bool* eos) {
+ if (_limit != -1 and _num_rows_returned + block->rows() >= _limit) {
+ block->set_num_rows(_limit - _num_rows_returned);
+ *eos = true;
+ }
+
+ _num_rows_returned += block->rows();
+ if (*eos) COUNTER_SET(_rows_returned_counter, _num_rows_returned);
+}
+
/*
Status ExecNode::enable_deny_reservation_debug_action() {
DCHECK_EQ(debug_action_, TDebugAction::SET_DENY_RESERVATION_PROBABILITY);
diff --git a/be/src/exec/exec_node.h b/be/src/exec/exec_node.h
index c58f862..7cad500 100644
--- a/be/src/exec/exec_node.h
+++ b/be/src/exec/exec_node.h
@@ -33,6 +33,8 @@
#include "util/runtime_profile.h"
#include "util/uid_util.h" // for print_id
+#include "vec/exprs/vexpr_context.h"
+
namespace doris {
class Expr;
class ExprContext;
@@ -219,6 +221,15 @@ protected:
/// fails.
Status release_unused_reservation();
+ /// Release all memory of block which got from child. The block
+ // 1. clear mem of valid column get from child, make sure child can reuse the mem
+ // 2. delete and release the column which create by function all and other reason
+ void release_block_memory(vectorized::Block& block, uint16_t child_idx = 0);
+
+ /// Only use in vectorized exec engine to check whether reach limit and cut num row for block
+ // and add block rows for profile
+ void reached_limit(vectorized::Block* block, bool* eos);
+
/// Enable the increase reservation denial probability on 'buffer_pool_client_' based on
/// the 'debug_action_' set on this node. Returns an error if 'debug_action_param_' is
/// invalid.
@@ -275,6 +286,8 @@ protected:
std::vector<ExprContext*> _conjunct_ctxs;
std::vector<TupleId> _tuple_ids;
+ std::unique_ptr<doris::vectorized::VExprContext*> _vconjunct_ctx_ptr;
+
std::vector<ExecNode*> _children;
RowDescriptor _row_descriptor;
diff --git a/be/src/exec/mysql_scan_node.h b/be/src/exec/mysql_scan_node.h
index 6d9d602..ff5fb54 100644
--- a/be/src/exec/mysql_scan_node.h
+++ b/be/src/exec/mysql_scan_node.h
@@ -63,7 +63,7 @@ private:
// The Mysql value is converted into the appropriate target type.
Status write_text_slot(char* value, int value_length, SlotDescriptor* slot,
RuntimeState* state);
-
+protected:
bool _is_init;
MysqlScannerParam _my_param;
// Name of Mysql table
diff --git a/be/src/exec/odbc_scan_node.cpp b/be/src/exec/odbc_scan_node.cpp
index 6bdc620..ea32cd8 100644
--- a/be/src/exec/odbc_scan_node.cpp
+++ b/be/src/exec/odbc_scan_node.cpp
@@ -29,9 +29,11 @@
namespace doris {
-OdbcScanNode::OdbcScanNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs)
+OdbcScanNode::OdbcScanNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs,
+ std::string scan_node_type)
: ScanNode(pool, tnode, descs),
_is_init(false),
+ _scan_node_type(scan_node_type),
_table_name(tnode.odbc_scan_node.table_name),
_connect_string(std::move(tnode.odbc_scan_node.connect_string)),
_query_string(std::move(tnode.odbc_scan_node.query_string)),
@@ -42,7 +44,7 @@ OdbcScanNode::OdbcScanNode(ObjectPool* pool, const TPlanNode& tnode, const Descr
OdbcScanNode::~OdbcScanNode() {}
Status OdbcScanNode::prepare(RuntimeState* state) {
- VLOG_CRITICAL << "OdbcScanNode::Prepare";
+ VLOG_CRITICAL << _scan_node_type << "::Prepare";
if (_is_init) {
return Status::OK();
@@ -91,7 +93,7 @@ Status OdbcScanNode::prepare(RuntimeState* state) {
Status OdbcScanNode::open(RuntimeState* state) {
RETURN_IF_ERROR(ExecNode::open(state));
- VLOG_CRITICAL << "OdbcScanNode::Open";
+ VLOG_CRITICAL << _scan_node_type << "::Open";
if (nullptr == state) {
return Status::InternalError("input pointer is null.");
@@ -125,7 +127,7 @@ Status OdbcScanNode::write_text_slot(char* value, int value_length, SlotDescript
}
Status OdbcScanNode::get_next(RuntimeState* state, RowBatch* row_batch, bool* eos) {
- VLOG_CRITICAL << "OdbcScanNode::GetNext";
+ VLOG_CRITICAL << _scan_node_type << "::GetNext";
if (nullptr == state || nullptr == row_batch || nullptr == eos) {
return Status::InternalError("input is nullptr pointer");
@@ -240,7 +242,7 @@ Status OdbcScanNode::close(RuntimeState* state) {
void OdbcScanNode::debug_string(int indentation_level, std::stringstream* out) const {
*out << string(indentation_level * 2, ' ');
- *out << "OdbcScanNode(tupleid=" << _tuple_id << " table=" << _table_name;
+ *out << _scan_node_type << "(tupleid=" << _tuple_id << " table=" << _table_name;
*out << ")" << std::endl;
for (int i = 0; i < _children.size(); ++i) {
diff --git a/be/src/exec/odbc_scan_node.h b/be/src/exec/odbc_scan_node.h
index 393d9ac..1754993 100644
--- a/be/src/exec/odbc_scan_node.h
+++ b/be/src/exec/odbc_scan_node.h
@@ -35,7 +35,8 @@ class Status;
class OdbcScanNode : public ScanNode {
public:
- OdbcScanNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs);
+ OdbcScanNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs,
+ std::string scan_node_type = "OdbcScanNode");
~OdbcScanNode();
// initialize _odbc_scanner, and create _text_converter.
@@ -53,6 +54,12 @@ public:
// No use
virtual Status set_scan_ranges(const std::vector<TScanRangeParams>& scan_ranges);
+ const TupleDescriptor* get_tuple_desc() { return _tuple_desc; }
+ TextConverter* get_text_converter() { return _text_converter.get(); }
+ ODBCConnector* get_odbc_scanner() { return _odbc_scanner.get(); }
+ const std::string& get_scan_node_type() { return _scan_node_type; }
+
+ bool is_init() { return _is_init; }
protected:
// Write debug string of this into out.
@@ -65,6 +72,9 @@ private:
RuntimeState* state);
bool _is_init;
+
+ std::string _scan_node_type;
+
// Name of Odbc table
std::string _table_name;
diff --git a/be/src/exec/olap_scan_node.cpp b/be/src/exec/olap_scan_node.cpp
index 5e4ba33..4c6abc1 100644
--- a/be/src/exec/olap_scan_node.cpp
+++ b/be/src/exec/olap_scan_node.cpp
@@ -61,7 +61,7 @@ OlapScanNode::OlapScanNode(ObjectPool* pool, const TPlanNode& tnode, const Descr
Status OlapScanNode::init(const TPlanNode& tnode, RuntimeState* state) {
RETURN_IF_ERROR(ExecNode::init(tnode, state));
- _direct_conjunct_size = state->enable_vectorized_exec() ? 1 : _conjunct_ctxs.size();
+ _direct_conjunct_size = _conjunct_ctxs.size();
const TQueryOptions& query_options = state->query_options();
if (query_options.__isset.max_scan_key_num) {
@@ -462,7 +462,6 @@ Status OlapScanNode::start_scan(RuntimeState* state) {
VLOG_CRITICAL << "Filter idle conjuncts";
// 5. Filter idle conjunct which already trans to olap filters
// this must be after build_scan_key, it will free the StringValue memory
- // TODO: filter idle conjunct in vexpr_contexts
remove_pushed_conjuncts(state);
VLOG_CRITICAL << "StartScanThread";
@@ -514,6 +513,7 @@ void OlapScanNode::remove_pushed_conjuncts(RuntimeState* state) {
_conjunct_ctxs = std::move(new_conjunct_ctxs);
_direct_conjunct_size = new_direct_conjunct_size;
+ // TODO: support vbloom_filter_predicate/vbinary_predicate and merge unpushed predicate to _vconjunct_ctx
for (auto push_down_ctx : _pushed_conjuncts_index) {
auto iter = _conjunctid_to_runtime_filter_ctxs.find(push_down_ctx);
if (iter != _conjunctid_to_runtime_filter_ctxs.end()) {
@@ -522,7 +522,13 @@ void OlapScanNode::remove_pushed_conjuncts(RuntimeState* state) {
}
// set vconjunct_ctx is empty, if all conjunct
if (_direct_conjunct_size == 0) {
+ if (_vconjunct_ctx_ptr.get() != nullptr) {
+ (*_vconjunct_ctx_ptr.get())->close(state);
+ _vconjunct_ctx_ptr = nullptr;
+ }
}
+ // filter idle conjunct in vexpr_contexts
+ _peel_pushed_conjuncts();
}
void OlapScanNode::eval_const_conjuncts() {
@@ -841,11 +847,6 @@ static bool ignore_cast(SlotDescriptor* slot, Expr* expr) {
bool OlapScanNode::should_push_down_in_predicate(doris::SlotDescriptor* slot,
doris::InPredicate* pred) {
- if (pred->is_not_in()) {
- // can not push down NOT IN predicate to storage engine
- return false;
- }
-
if (Expr::type_without_cast(pred->get_child(0)) != TExprNodeType::SLOT_REF) {
// not a slot ref(column)
return false;
@@ -1670,4 +1671,46 @@ Status OlapScanNode::add_one_batch(RowBatch* row_batch) {
}
+vectorized::VExpr* OlapScanNode::_dfs_peel_conjunct(vectorized::VExpr* expr, int& leaf_index) {
+ static constexpr auto is_leaf = [](vectorized::VExpr* expr) { return !expr->is_and_expr(); };
+
+ if (is_leaf(expr)) {
+ return _pushed_conjuncts_index.count(leaf_index++) ? nullptr : expr;
+ } else {
+ vectorized::VExpr* left_child = _dfs_peel_conjunct(expr->children()[0], leaf_index);
+ vectorized::VExpr* right_child = _dfs_peel_conjunct(expr->children()[1], leaf_index);
+
+ if (left_child != nullptr && right_child != nullptr) {
+ expr->set_children({left_child, right_child});
+ return expr;
+ }
+ // here do not close Expr* now
+ return left_child != nullptr ? left_child : right_child;
+ }
+}
+
+// This function is used to remove pushed expr in expr tree.
+// It relies on the logic of function convertConjunctsToAndCompoundPredicate() of FE splicing expr.
+// It requires FE to satisfy each splicing with 'and' expr, and spliced from left to right, in order.
+// Expr tree specific forms do not require requirements.
+void OlapScanNode::_peel_pushed_conjuncts() {
+ if (_vconjunct_ctx_ptr.get() == nullptr) return;
+
+ int leaf_index = 0;
+ vectorized::VExpr* conjunct_expr_root = (*_vconjunct_ctx_ptr.get())->root();
+
+ if (conjunct_expr_root != nullptr) {
+ vectorized::VExpr* new_conjunct_expr_root =
+ _dfs_peel_conjunct(conjunct_expr_root, leaf_index);
+ if (new_conjunct_expr_root == nullptr) {
+ _vconjunct_ctx_ptr = nullptr;
+ _scanner_profile->add_info_string("VconjunctExprTree", "null");
+ } else {
+ (*_vconjunct_ctx_ptr.get())->set_root(new_conjunct_expr_root);
+ _scanner_profile->add_info_string("VconjunctExprTree",
+ new_conjunct_expr_root->debug_string());
+ }
+ }
+}
+
} // namespace doris
diff --git a/be/src/exec/olap_scan_node.h b/be/src/exec/olap_scan_node.h
index 3b7df03..f9905e2 100644
--- a/be/src/exec/olap_scan_node.h
+++ b/be/src/exec/olap_scan_node.h
@@ -32,6 +32,8 @@
#include "runtime/vectorized_row_batch.h"
#include "util/progress_updater.h"
#include "util/spinlock.h"
+#include "vec/exec/volap_scanner.h"
+#include "vec/exprs/vexpr.h"
namespace doris {
class IRuntimeFilter;
@@ -158,6 +160,7 @@ protected:
RuntimeProfile* profile);
friend class OlapScanner;
+ friend class doris::vectorized::VOlapScanner;
// Tuple id resolved in prepare() to set _tuple_desc;
TupleId _tuple_id;
@@ -215,7 +218,7 @@ protected:
std::mutex _scan_batches_lock;
std::condition_variable _scan_batch_added_cv;
- int64_t _running_thread = 0;
+ std::atomic_int _running_thread = 0;
std::condition_variable _scan_thread_exit_cv;
std::list<RowBatch*> _scan_row_batches;
@@ -321,6 +324,9 @@ protected:
RuntimeProfile::Counter* _scanner_wait_worker_timer = nullptr;
RuntimeProfile::Counter* _olap_wait_batch_queue_timer = nullptr;
+
+ vectorized::VExpr* _dfs_peel_conjunct(vectorized::VExpr* expr, int& leaf_index);
+ void _peel_pushed_conjuncts(); // remove pushed expr from conjunct tree
};
} // namespace doris
diff --git a/be/src/exec/olap_scanner.cpp b/be/src/exec/olap_scanner.cpp
index 2e3af6b..8e3c3c9 100644
--- a/be/src/exec/olap_scanner.cpp
+++ b/be/src/exec/olap_scanner.cpp
@@ -19,9 +19,9 @@
#include <string>
-#include "gen_cpp/PaloInternalService_types.h"
#include "common/utils.h"
#include "exprs/expr_context.h"
+#include "gen_cpp/PaloInternalService_types.h"
#include "olap/decimal12.h"
#include "olap/field.h"
#include "olap/uint24.h"
@@ -183,8 +183,10 @@ Status OlapScanner::_init_params(
_params.rs_readers[1]->rowset()->start_version() == 2 &&
!_params.rs_readers[1]->rowset()->rowset_meta()->is_segments_overlapping());
+ _params.origin_return_columns = &_return_columns;
if (_aggregation || single_version) {
_params.return_columns = _return_columns;
+ _params.direct_mode = true;
} else {
// we need to fetch all key columns to do the right aggregation on storage engine side.
for (size_t i = 0; i < _tablet->num_key_columns(); ++i) {
@@ -246,7 +248,8 @@ Status OlapScanner::_init_return_columns() {
}
}
if (auto sequence_col_idx = _tablet->tablet_schema().sequence_col_idx();
- has_replace_col && std::find(_return_columns.begin(), _return_columns.end(), sequence_col_idx) == _return_columns.end()) {
+ has_replace_col && std::find(_return_columns.begin(), _return_columns.end(),
+ sequence_col_idx) == _return_columns.end()) {
_return_columns.push_back(sequence_col_idx);
}
}
diff --git a/be/src/exec/partitioned_aggregation_node.cc b/be/src/exec/partitioned_aggregation_node.cc
index 2311f82..68f536f 100644
--- a/be/src/exec/partitioned_aggregation_node.cc
+++ b/be/src/exec/partitioned_aggregation_node.cc
@@ -1048,7 +1048,7 @@ Tuple* PartitionedAggregationNode::GetOutputTuple(const vector<NewAggFnEvaluator
}
if (needs_finalize_) {
NewAggFnEvaluator::Finalize(agg_fn_evals, tuple, dst,
- grouping_exprs_.size() == 0 && child(0)->rows_returned() == 0);
+ grouping_exprs_.size() == 0 && child(0)->rows_returned() == 0);
} else {
NewAggFnEvaluator::Serialize(agg_fn_evals, tuple);
}
diff --git a/be/src/exec/scan_node.h b/be/src/exec/scan_node.h
index 5f8ec4b..bd09991 100644
--- a/be/src/exec/scan_node.h
+++ b/be/src/exec/scan_node.h
@@ -91,7 +91,7 @@ public:
protected:
RuntimeProfile::Counter* _bytes_read_counter; // # bytes read from the scanner
- // # rows/tuples read from the scanner (including those discarded by eval_conjucts())
+ // # rows/tuples read from the scanner (including those discarded by eval_conjuncts())
RuntimeProfile::Counter* _rows_read_counter;
// Wall based aggregate read throughput [bytes/sec]
RuntimeProfile::Counter* _total_throughput_counter;
diff --git a/be/src/exec/schema_scan_node.h b/be/src/exec/schema_scan_node.h
index 2052de0..f45086c 100644
--- a/be/src/exec/schema_scan_node.h
+++ b/be/src/exec/schema_scan_node.h
@@ -62,7 +62,7 @@ private:
void debug_string(int indentation_level, std::stringstream* out) const override;
// Copy one row from schema table to input tuple
void copy_one_row();
-
+protected:
bool _is_init;
const std::string _table_name;
SchemaScannerParam _scanner_param;
diff --git a/be/src/exec/tablet_info.cpp b/be/src/exec/tablet_info.cpp
index 7d470bf..a1195d5 100644
--- a/be/src/exec/tablet_info.cpp
+++ b/be/src/exec/tablet_info.cpp
@@ -407,4 +407,228 @@ uint32_t OlapTablePartitionParam::_compute_dist_hash(Tuple* key) const {
return hash_val;
}
+VOlapTablePartitionParam::VOlapTablePartitionParam(std::shared_ptr<OlapTableSchemaParam>& schema,
+ const TOlapTablePartitionParam& t_param)
+ : _schema(schema),
+ _t_param(t_param),
+ _slots(_schema->tuple_desc()->slots()),
+ _mem_tracker(MemTracker::CreateTracker(-1, "OlapTablePartitionParam")) {
+ for (auto slot : _slots) {
+ _partition_block.insert({slot->get_empty_mutable_column(), slot->get_data_type_ptr(), slot->col_name()});
+ }
+}
+
+VOlapTablePartitionParam::~VOlapTablePartitionParam() {
+ _mem_tracker->Release(_mem_usage);
+}
+
+Status VOlapTablePartitionParam::init() {
+ std::vector<std::string> slot_column_names;
+ for (auto slot_desc : _schema->tuple_desc()->slots()) {
+ slot_column_names.emplace_back(slot_desc->col_name());
+ }
+
+ auto find_slot_locs = [&slot_column_names](const std::string& slot_name, std::vector<uint16_t>& locs, const std::string& column_type) {
+ auto it = std::find(slot_column_names.begin(), slot_column_names.end(), slot_name);
+ if (it == slot_column_names.end()) {
+ return Status::InternalError(column_type + " column not found, column =" + slot_name);
+ }
+ locs.emplace_back(it - slot_column_names.begin());
+ return Status::OK();
+ };
+
+ if (_t_param.__isset.partition_columns) {
+ for (auto& part_col : _t_param.partition_columns) {
+ RETURN_IF_ERROR(find_slot_locs(part_col, _partition_slot_locs, "partition"));
+ }
+ }
+
+ _partitions_map.reset(new std::map<BlockRow*, VOlapTablePartition*, VOlapTablePartKeyComparator>(
+ VOlapTablePartKeyComparator(_partition_slot_locs)));
+ if (_t_param.__isset.distributed_columns) {
+ for (auto& col : _t_param.distributed_columns) {
+ RETURN_IF_ERROR(find_slot_locs(col, _distributed_slot_locs, "distributed"));
+ }
+ }
+
+ DCHECK(!_t_param.partitions.empty()) << "must have at least 1 partition";
+ _is_in_partition = _t_param.partitions[0].__isset.in_keys;
+
+ // initial partitions
+ for (int i = 0; i < _t_param.partitions.size(); ++i) {
+ const TOlapTablePartition& t_part = _t_param.partitions[i];
+ auto part = _obj_pool.add(new VOlapTablePartition(&_partition_block));
+ part->id = t_part.id;
+
+ if (!_is_in_partition) {
+ if (t_part.__isset.start_keys) {
+ RETURN_IF_ERROR(_create_partition_keys(t_part.start_keys, &part->start_key));
+ }
+
+ if (t_part.__isset.end_keys) {
+ RETURN_IF_ERROR(_create_partition_keys(t_part.end_keys, &part->end_key));
+ }
+ } else {
+ for (const auto& keys : t_part.in_keys) {
+ RETURN_IF_ERROR(_create_partition_keys(keys, &part->in_keys.emplace_back(&_partition_block, -1)));
+ }
+ }
+
+ part->num_buckets = t_part.num_buckets;
+ auto num_indexes = _schema->indexes().size();
+ if (t_part.indexes.size() != num_indexes) {
+ std::stringstream ss;
+ ss << "number of partition's index is not equal with schema's"
+ << ", num_part_indexes=" << t_part.indexes.size()
+ << ", num_schema_indexes=" << num_indexes;
+ return Status::InternalError(ss.str());
+ }
+ part->indexes = t_part.indexes;
+ std::sort(part->indexes.begin(), part->indexes.end(),
+ [](const OlapTableIndexTablets& lhs, const OlapTableIndexTablets& rhs) {
+ return lhs.index_id < rhs.index_id;
+ });
+ // check index
+ for (int j = 0; j < num_indexes; ++j) {
+ if (part->indexes[j].index_id != _schema->indexes()[j]->index_id) {
+ std::stringstream ss;
+ ss << "partition's index is not equal with schema's"
+ << ", part_index=" << part->indexes[j].index_id
+ << ", schema_index=" << _schema->indexes()[j]->index_id;
+ return Status::InternalError(ss.str());
+ }
+ }
+ _partitions.emplace_back(part);
+ if (_is_in_partition) {
+ for (auto& in_key : part->in_keys) {
+ _partitions_map->emplace(&in_key, part);
+ }
+ } else {
+ _partitions_map->emplace(&part->end_key, part);
+ }
+ }
+
+ _mem_usage = _partition_block.allocated_bytes();
+ _mem_tracker->Consume(_mem_usage);
+ return Status::OK();
+}
+
+bool VOlapTablePartitionParam::find_tablet(BlockRow* block_row, const VOlapTablePartition** partition,
+ uint32_t* dist_hashes) const {
+ auto it = _is_in_partition ? _partitions_map->find(block_row) : _partitions_map->upper_bound(block_row);
+ if (it == _partitions_map->end()) {
+ return false;
+ }
+ if (_is_in_partition || _part_contains(it->second, block_row)) {
+ *partition = it->second;
+ *dist_hashes = _compute_dist_hash(block_row);
+ return true;
+ }
+ return false;
+}
+
+Status VOlapTablePartitionParam::_create_partition_keys(const std::vector<TExprNode>& t_exprs,
+ BlockRow* part_key) {
+ for (int i = 0; i < t_exprs.size(); i++) {
+ RETURN_IF_ERROR(_create_partition_key(t_exprs[i], part_key->first,
+ _partition_slot_locs[i]));
+ }
+ return Status::OK();
+}
+
+Status VOlapTablePartitionParam::_create_partition_key(const TExprNode& t_expr, vectorized::Block* block,
+ uint16_t pos) {
+ auto column = std::move(*block->get_by_position(pos).column).mutate();
+ switch (t_expr.node_type) {
+ case TExprNodeType::DATE_LITERAL: {
+ vectorized::VecDateTimeValue dt;
+ if (!dt.from_date_str(
+ t_expr.date_literal.value.c_str(), t_expr.date_literal.value.size())) {
+ std::stringstream ss;
+ ss << "invalid date literal in partition column, date=" << t_expr.date_literal;
+ return Status::InternalError(ss.str());
+ }
+ column->insert_data(reinterpret_cast<const char *>(&dt), 0);
+ break;
+ }
+ case TExprNodeType::INT_LITERAL: {
+ switch (t_expr.type.types[0].scalar_type.type) {
+ case TPrimitiveType::TINYINT: {
+ int8_t value = t_expr.int_literal.value;
+ column->insert_data(reinterpret_cast<const char *>(&value), 0);
+ break;
+ }
+ case TPrimitiveType::SMALLINT: {
+ int16_t value = t_expr.int_literal.value;
+ column->insert_data(reinterpret_cast<const char *>(&value), 0);
+ break;
+ }
+ case TPrimitiveType::INT: {
+ int32_t value = t_expr.int_literal.value;
+ column->insert_data(reinterpret_cast<const char *>(&value), 0);
+ break;
+ }
+ default:
+ int64_t value = t_expr.int_literal.value;
+ column->insert_data(reinterpret_cast<const char *>(&value), 0);
+ }
+ break;
+ } case TExprNodeType::LARGE_INT_LITERAL: {
+ StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS;
+ __int128 value = StringParser::string_to_int<__int128>(t_expr.large_int_literal.value.c_str(),
+ t_expr.large_int_literal.value.size(),
+ &parse_result);
+ if (parse_result != StringParser::PARSE_SUCCESS) {
+ value = MAX_INT128;
+ }
+ column->insert_data(reinterpret_cast<const char *>(&value), 0);
+ break;
+ } case TExprNodeType::STRING_LITERAL: {
+ int len = t_expr.string_literal.value.size();
+ const char* str_val = t_expr.string_literal.value.c_str();
+
+ // CHAR is a fixed-length string and needs to use the length in the slot definition,
+ // VARVHAR is a variable-length string and needs to use the length of the string itself
+ // padding 0 to CHAR field
+// if (TYPE_CHAR == slot_desc->type().type && len < slot_desc->type().len) {
+// auto new_ptr = (char*)_mem_pool->allocate(slot_desc->type().len);
+// memset(new_ptr, 0, slot_desc->type().len);
+// memcpy(new_ptr, str_val, len);
+//
+// str_val = new_ptr;
+// len = slot_desc->type().len;
+// }
+ column->insert_data(str_val, len);
+ break;
+ } case TExprNodeType::BOOL_LITERAL: {
+ column->insert_data(reinterpret_cast<const char *>(&t_expr.bool_literal.value), 0);
+ break;
+ } default: {
+ std::stringstream ss;
+ ss << "unsupported partition column node type, type=" << t_expr.node_type;
+ return Status::InternalError(ss.str());
+ }
+ }
+ return Status::OK();
+}
+
+uint32_t VOlapTablePartitionParam::_compute_dist_hash(BlockRow* key) const {
+ uint32_t hash_val = 0;
+ for (int i = 0; i < _distributed_slot_locs.size(); ++i) {
+ auto slot_desc = _slots[_distributed_slot_locs[i]];
+ auto column = key->first->get_by_position(_distributed_slot_locs[i]).column;
+
+ auto val = column->get_data_at(key->second);
+ if (val.data != nullptr) {
+ hash_val = RawValue::zlib_crc32(val.data, val.size, slot_desc->type().type, hash_val);
+ } else {
+ // NULL is treat as 0 when hash
+ static const int INT_VALUE = 0;
+ static const TypeDescriptor INT_TYPE(TYPE_INT);
+ hash_val = RawValue::zlib_crc32(&INT_VALUE, INT_TYPE, hash_val);
+ }
+ }
+ return hash_val;
+}
+
} // namespace doris
diff --git a/be/src/exec/tablet_info.h b/be/src/exec/tablet_info.h
index 7bb2640..53221ba 100644
--- a/be/src/exec/tablet_info.h
+++ b/be/src/exec/tablet_info.h
@@ -31,6 +31,8 @@
#include "runtime/raw_value.h"
#include "runtime/tuple.h"
+#include "vec/core/block.h"
+
namespace doris {
class MemPool;
@@ -205,6 +207,104 @@ private:
_partitions_map;
};
+using BlockRow = std::pair<vectorized::Block*, int32_t>;
+
+struct VOlapTablePartition {
+ int64_t id = 0;
+ BlockRow start_key;
+ BlockRow end_key;
+ std::vector<BlockRow> in_keys;
+ int64_t num_buckets = 0;
+ std::vector<OlapTableIndexTablets> indexes;
+
+ VOlapTablePartition(vectorized::Block* partition_block):
+ start_key{partition_block, -1}, end_key{partition_block, -1} {};
+};
+
+class VOlapTablePartKeyComparator {
+public:
+ VOlapTablePartKeyComparator(const std::vector<uint16_t>& slot_locs)
+ : _slot_locs(slot_locs) {}
+
+ // return true if lhs < rhs
+ // 'row' is -1 mean
+ bool operator()(const BlockRow* lhs, const BlockRow* rhs) const {
+ if (lhs->second == -1) {
+ return false;
+ } else if (rhs->second == -1) {
+ return true;
+ }
+
+ for (auto slot_loc : _slot_locs) {
+ auto res = lhs->first->get_by_position(slot_loc).column->compare_at(
+ lhs->second, rhs->second, *rhs->first->get_by_position(slot_loc).column, -1);
+ if (res != 0) {
+ return res < 0;
+ }
+ }
+ // equal, return false
+ return false;
+ }
+
+private:
+ const std::vector<uint16_t>& _slot_locs;
+};
+
+// store an olap table's tablet information
+class VOlapTablePartitionParam {
+public:
+ VOlapTablePartitionParam(std::shared_ptr<OlapTableSchemaParam>& schema,
+ const TOlapTablePartitionParam& param);
+
+ ~VOlapTablePartitionParam();
+
+ Status init();
+
+ int64_t db_id() const { return _t_param.db_id; }
+ int64_t table_id() const { return _t_param.table_id; }
+ int64_t version() const { return _t_param.version; }
+
+ // return true if we found this tuple in partition
+ bool find_tablet(BlockRow* block_row, const VOlapTablePartition** partitions,
+ uint32_t* dist_hash) const;
+
+ const std::vector<VOlapTablePartition*>& get_partitions() const { return _partitions; }
+
+private:
+ Status _create_partition_keys(const std::vector<TExprNode>& t_exprs, BlockRow* part_key);
+
+ Status _create_partition_key(const TExprNode& t_expr, vectorized::Block* block, uint16_t pos);
+
+ uint32_t _compute_dist_hash(BlockRow* key) const;
+
+ // check if this partition contain this key
+ bool _part_contains(VOlapTablePartition* part, BlockRow* key) const {
+ // start_key.second == -1 means only single partition
+ VOlapTablePartKeyComparator comparator(_partition_slot_locs);
+ return part->start_key.second == -1 || !comparator(key, &part->start_key);
+ }
+
+private:
+ // this partition only valid in this schema
+ std::shared_ptr<OlapTableSchemaParam> _schema;
+ TOlapTablePartitionParam _t_param;
+
+ const std::vector<SlotDescriptor*>& _slots;
+ std::vector<uint16_t> _partition_slot_locs;
+ std::vector<uint16_t> _distributed_slot_locs;
+
+ ObjectPool _obj_pool;
+ vectorized::Block _partition_block;
+ std::shared_ptr<MemTracker> _mem_tracker;
+ std::vector<VOlapTablePartition*> _partitions;
+ std::unique_ptr<std::map<BlockRow*, VOlapTablePartition*, VOlapTablePartKeyComparator>>
+ _partitions_map;
+
+ bool _is_in_partition = false;
+ uint32_t _mem_usage = 0;
+};
+
+
using TabletLocation = TTabletLocation;
// struct TTabletLocation {
// 1: required i64 tablet_id
diff --git a/be/src/exec/tablet_sink.cpp b/be/src/exec/tablet_sink.cpp
index 184c4b0..be1650b 100644
--- a/be/src/exec/tablet_sink.cpp
+++ b/be/src/exec/tablet_sink.cpp
@@ -249,6 +249,53 @@ Status NodeChannel::add_row(Tuple* input_tuple, int64_t tablet_id) {
return Status::OK();
}
+Status NodeChannel::add_row(BlockRow& block_row, int64_t tablet_id) {
+ // If add_row() when _eos_is_produced==true, there must be sth wrong, we can only mark this channel as failed.
+ auto st = none_of({_cancelled, _eos_is_produced});
+ if (!st.ok()) {
+ if (_cancelled) {
+ std::lock_guard<SpinLock> l(_cancel_msg_lock);
+ return Status::InternalError("add row failed. " + _cancel_msg);
+ } else {
+ return st.clone_and_prepend("already stopped, can't add row. cancelled/eos: ");
+ }
+ }
+
+ // We use OlapTableSink mem_tracker which has the same ancestor of _plan node,
+ // so in the ideal case, mem limit is a matter for _plan node.
+ // But there is still some unfinished things, we do mem limit here temporarily.
+ // _cancelled may be set by rpc callback, and it's possible that _cancelled might be set in any of the steps below.
+ // It's fine to do a fake add_row() and return OK, because we will check _cancelled in next add_row() or mark_close().
+ while (!_cancelled && _parent->_mem_tracker->AnyLimitExceeded(MemLimit::HARD) &&
+ _pending_batches_num > 0) {
+ SCOPED_ATOMIC_TIMER(&_mem_exceeded_block_ns);
+ SleepFor(MonoDelta::FromMilliseconds(10));
+ }
+
+ auto row_no = _cur_batch->add_row();
+ if (row_no == RowBatch::INVALID_ROW_INDEX) {
+ {
+ SCOPED_ATOMIC_TIMER(&_queue_push_lock_ns);
+ std::lock_guard<std::mutex> l(_pending_batches_lock);
+ //To simplify the add_row logic, postpone adding batch into req until the time of sending req
+ _pending_batches.emplace(std::move(_cur_batch), _cur_add_batch_request);
+ _pending_batches_num++;
+ }
+
+ _cur_batch.reset(new RowBatch(*_row_desc, _batch_size, _parent->_mem_tracker.get()));
+ _cur_add_batch_request.clear_tablet_ids();
+
+ row_no = _cur_batch->add_row();
+ }
+ DCHECK_NE(row_no, RowBatch::INVALID_ROW_INDEX);
+
+ _cur_batch->get_row(row_no)->set_tuple(0,
+ block_row.first->deep_copy_tuple(*_tuple_desc, _cur_batch->tuple_data_pool(), block_row.second, 0, true));
+ _cur_batch->commit_last_row();
+ _cur_add_batch_request.add_tablet_ids(tablet_id);
+ return Status::OK();
+}
+
Status NodeChannel::mark_close() {
auto st = none_of({_cancelled, _eos_is_produced});
if (!st.ok()) {
@@ -499,6 +546,29 @@ Status IndexChannel::add_row(Tuple* tuple, int64_t tablet_id) {
return Status::OK();
}
+Status IndexChannel::add_row(BlockRow& block_row, int64_t tablet_id) {
+ auto it = _channels_by_tablet.find(tablet_id);
+ DCHECK(it != _channels_by_tablet.end()) << "unknown tablet, tablet_id=" << tablet_id;
+ std::stringstream ss;
+ for (auto channel : it->second) {
+ // if this node channel is already failed, this add_row will be skipped
+ auto st = channel->add_row(block_row, tablet_id);
+ if (!st.ok()) {
+ mark_as_failed(channel);
+ ss << st.get_error_msg() << "; ";
+ }
+ }
+
+ if (has_intolerable_failure()) {
+ std::stringstream ss2;
+ ss2 << "index channel has intolerable failure. " << BackendOptions::get_localhost()
+ << ", err: " << ss.str();
+ return Status::InternalError(ss2.str());
+ }
+
+ return Status::OK();
+}
+
bool IndexChannel::has_intolerable_failure() {
for (const auto& it : _failed_channels) {
if (it.second.size() >= ((_parent->_num_replicas + 1) / 2)) {
@@ -728,7 +798,7 @@ Status OlapTableSink::send(RuntimeState* state, RowBatch* input_batch) {
batch = _output_batch.get();
}
int num_invalid_rows = 0;
- if (_need_validate_data) {
+ {
SCOPED_RAW_TIMER(&_validate_data_ns);
_filter_bitmap.Reset(batch->num_rows());
num_invalid_rows = _validate_data(state, batch, &_filter_bitmap);
diff --git a/be/src/exec/tablet_sink.h b/be/src/exec/tablet_sink.h
index 261912d..6a2b96a 100644
--- a/be/src/exec/tablet_sink.h
+++ b/be/src/exec/tablet_sink.h
@@ -160,6 +160,8 @@ public:
Status add_row(Tuple* tuple, int64_t tablet_id);
+ Status add_row(BlockRow& block_row, int64_t tablet_id);
+
// two ways to stop channel:
// 1. mark_close()->close_wait() PS. close_wait() will block waiting for the last AddBatch rpc response.
// 2. just cancel()
@@ -277,6 +279,8 @@ public:
Status add_row(Tuple* tuple, int64_t tablet_id);
+ Status add_row(BlockRow& block_row, int64_t tablet_id);
+
void for_each_node_channel(const std::function<void(NodeChannel*)>& func) {
for (auto& it : _node_channels) {
func(it.second);
@@ -351,7 +355,7 @@ private:
// only focus on pending batches and channel status, the internal errors of NodeChannels will be handled by the producer
void _send_batch_process();
-private:
+protected:
friend class NodeChannel;
friend class IndexChannel;
diff --git a/be/src/exec/text_converter.cpp b/be/src/exec/text_converter.cpp
index 7801f66..e2d37d5 100644
--- a/be/src/exec/text_converter.cpp
+++ b/be/src/exec/text_converter.cpp
@@ -59,4 +59,26 @@ void TextConverter::unescape_string(const char* src, char* dest, size_t* len) {
*len = dest_ptr - dest_start;
}
+void TextConverter::unescape_string_on_spot(const char* src, size_t* len) {
+ char* dest_ptr = const_cast<char*>(src);
+ const char* end = src + *len;
+ bool escape_next_char = false;
+
+ while (src < end) {
+ if (*src == _escape_char) {
+ escape_next_char = !escape_next_char;
+ } else {
+ escape_next_char = false;
+ }
+
+ if (escape_next_char) {
+ ++src;
+ } else {
+ *dest_ptr++ = *src++;
+ }
+ }
+
+ *len = dest_ptr - src;
+}
+
} // namespace doris
diff --git a/be/src/exec/text_converter.h b/be/src/exec/text_converter.h
index 4d97b7c..36b3254 100644
--- a/be/src/exec/text_converter.h
+++ b/be/src/exec/text_converter.h
@@ -19,7 +19,7 @@
#define DORIS_BE_SRC_QUERY_EXEC_TEXT_CONVERTER_H
#include "runtime/runtime_state.h"
-
+#include "vec/core/block.h"
namespace doris {
class MemPool;
@@ -48,11 +48,14 @@ public:
bool write_slot(const SlotDescriptor* slot_desc, Tuple* tuple, const char* data, int len,
bool copy_string, bool need_escape, MemPool* pool);
+ bool write_column(const SlotDescriptor* slot_desc, vectorized::MutableColumnPtr* column_ptr,
+ const char* data, size_t len, bool copy_string, bool need_escape);
+
// Removes escape characters from len characters of the null-terminated string src,
// and copies the unescaped string into dest, changing *len to the unescaped length.
// No null-terminator is added to dest.
void unescape_string(const char* src, char* dest, size_t* len);
-
+ void unescape_string_on_spot(const char* src, size_t* len);
// Removes escape characters from 'str', allocating a new string from pool.
// 'str' is updated with the new ptr and length.
void unescape_string(StringValue* str, MemPool* pool);
diff --git a/be/src/exec/text_converter.hpp b/be/src/exec/text_converter.hpp
index 1b1a60e..44e807c 100644
--- a/be/src/exec/text_converter.hpp
+++ b/be/src/exec/text_converter.hpp
@@ -17,6 +17,7 @@
#ifndef DORIS_BE_SRC_QUERY_EXEC_TEXT_CONVERTER_HPP
#define DORIS_BE_SRC_QUERY_EXEC_TEXT_CONVERTER_HPP
+#include <sql.h>
#include <boost/algorithm/string.hpp>
@@ -32,7 +33,7 @@
#include "util/binary_cast.hpp"
#include "util/string_parser.hpp"
#include "util/types.h"
-
+#include "vec/runtime/vdatetime_value.h"
namespace doris {
// Note: this function has a codegen'd version. Changing this function requires
@@ -165,6 +166,134 @@ inline bool TextConverter::write_slot(const SlotDescriptor* slot_desc, Tuple* tu
return true;
}
+inline bool TextConverter::write_column(const SlotDescriptor* slot_desc,
+ vectorized::MutableColumnPtr* column_ptr, const char* data,
+ size_t len, bool copy_string, bool need_escape) {
+ vectorized::IColumn* col_ptr = column_ptr->get();
+ // \N means it's NULL
+ if (true == slot_desc->is_nullable()) {
+ auto* nullable_column = reinterpret_cast<vectorized::ColumnNullable*>(column_ptr->get());
+ if ((len == 2 && data[0] == '\\' && data[1] == 'N') || len == SQL_NULL_DATA) {
+ nullable_column->insert_data(nullptr, 0);
+ return true;
+ } else {
+ nullable_column->get_null_map_data().push_back(0);
+ col_ptr = &nullable_column->get_nested_column();
+ }
+ }
+ StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS;
+
+ // Parse the raw-text data. Translate the text string to internal format.
+ switch (slot_desc->type().type) {
+ case TYPE_HLL:
+ case TYPE_VARCHAR:
+ case TYPE_CHAR: {
+ if (need_escape) {
+ unescape_string_on_spot(data, &len);
+ }
+ reinterpret_cast<vectorized::ColumnString*>(col_ptr)->insert_data(data, len);
+ break;
+ }
+
+ case TYPE_BOOLEAN: {
+ bool num = StringParser::string_to_bool(data, len, &parse_result);
+ reinterpret_cast<vectorized::ColumnVector<vectorized::UInt8>*>(col_ptr)->insert_value(
+ (uint8_t)num);
+ break;
+ }
+ case TYPE_TINYINT: {
+ int8_t num = StringParser::string_to_int<int8_t>(data, len, &parse_result);
+ reinterpret_cast<vectorized::ColumnVector<vectorized::Int8>*>(col_ptr)->insert_value(num);
+ break;
+ }
+ case TYPE_SMALLINT: {
+ int16_t num = StringParser::string_to_int<int16_t>(data, len, &parse_result);
+ reinterpret_cast<vectorized::ColumnVector<vectorized::Int16>*>(col_ptr)->insert_value(num);
+ break;
+ }
+ case TYPE_INT: {
+ int32_t num = StringParser::string_to_int<int32_t>(data, len, &parse_result);
+ reinterpret_cast<vectorized::ColumnVector<vectorized::Int32>*>(col_ptr)->insert_value(num);
+ break;
+ }
+ case TYPE_BIGINT: {
+ int64_t num = StringParser::string_to_int<int64_t>(data, len, &parse_result);
+ reinterpret_cast<vectorized::ColumnVector<vectorized::Int64>*>(col_ptr)->insert_value(num);
+ break;
+ }
+ case TYPE_LARGEINT: {
+ __int128 num = StringParser::string_to_int<__int128>(data, len, &parse_result);
+ reinterpret_cast<vectorized::ColumnVector<vectorized::Int128>*>(col_ptr)->insert_value(num);
+ break;
+ }
+
+ case TYPE_FLOAT: {
+ float num = StringParser::string_to_float<float>(data, len, &parse_result);
+ reinterpret_cast<vectorized::ColumnVector<vectorized::Float32>*>(col_ptr)->insert_value(
+ num);
+ break;
+ }
+ case TYPE_DOUBLE: {
+ double num = StringParser::string_to_float<double>(data, len, &parse_result);
+ reinterpret_cast<vectorized::ColumnVector<vectorized::Float64>*>(col_ptr)->insert_value(
+ num);
+ break;
+ }
+ case TYPE_DATE: {
+ vectorized::VecDateTimeValue ts_slot;
+ if (!ts_slot.from_date_str(data, len)) {
+ parse_result = StringParser::PARSE_FAILURE;
+ break;
+ }
+ ts_slot.cast_to_date();
+ reinterpret_cast<vectorized::ColumnVector<vectorized::Int64>*>(col_ptr)->insert_data(
+ reinterpret_cast<char*>(&ts_slot), 0);
+ break;
+ }
+
+ case TYPE_DATETIME: {
+ vectorized::VecDateTimeValue ts_slot;
+ if (!ts_slot.from_date_str(data, len)) {
+ parse_result = StringParser::PARSE_FAILURE;
+ break;
+ }
+ ts_slot.to_datetime();
+ reinterpret_cast<vectorized::ColumnVector<vectorized::Int64>*>(col_ptr)->insert_data(
+ reinterpret_cast<char*>(&ts_slot), 0);
+ break;
+ }
+
+ case TYPE_DECIMALV2: {
+ DecimalV2Value decimal_slot;
+ if (decimal_slot.parse_from_str(data, len)) {
+ parse_result = StringParser::PARSE_FAILURE;
+ break;
+ }
+ PackedInt128 num = binary_cast<DecimalV2Value, PackedInt128>(decimal_slot);
+ reinterpret_cast<vectorized::ColumnVector<doris::PackedInt128>*>(col_ptr)->insert_value(
+ num.value);
+ break;
+ }
+
+ default:
+ DCHECK(false) << "bad slot type: " << slot_desc->type();
+ break;
+ }
+
+ if (parse_result == StringParser::PARSE_FAILURE) {
+ if (true == slot_desc->is_nullable()) {
+ auto* nullable_column =
+ reinterpret_cast<vectorized::ColumnNullable*>(column_ptr->get());
+ size_t size = nullable_column->get_null_map_data().size();
+ doris::vectorized::NullMap& null_map_data = nullable_column->get_null_map_data();
+ null_map_data[size - 1] = 1;
+ } else {
+ return false;
+ }
+ }
+ return true;
+}
+
} // namespace doris
-#endif
+#endif
\ No newline at end of file
diff --git a/be/src/exprs/CMakeLists.txt b/be/src/exprs/CMakeLists.txt
index 5ebf1d1..5d69aa7 100644
--- a/be/src/exprs/CMakeLists.txt
+++ b/be/src/exprs/CMakeLists.txt
@@ -71,7 +71,6 @@ add_library(Exprs
hll_function.cpp
grouping_sets_functions.cpp
topn_function.cpp
-
table_function/explode_split.cpp
table_function/explode_bitmap.cpp
table_function/explode_json_array.cpp
diff --git a/be/src/exprs/aggregate_functions.cpp b/be/src/exprs/aggregate_functions.cpp
index 1f8db84..a22c3ee 100644
--- a/be/src/exprs/aggregate_functions.cpp
+++ b/be/src/exprs/aggregate_functions.cpp
@@ -592,30 +592,14 @@ void AggregateFunctions::sum(FunctionContext* ctx, const DecimalV2Val& src, Deci
}
if (dst->is_null) {
- dst->is_null = false;
- dst->set_to_zero();
+ init_zero_not_null<DecimalV2Val>(ctx, dst);
}
-
DecimalV2Value new_src = DecimalV2Value::from_decimal_val(src);
DecimalV2Value new_dst = DecimalV2Value::from_decimal_val(*dst);
new_dst = new_dst + new_src;
new_dst.to_decimal_val(dst);
}
-template <>
-void AggregateFunctions::sum(FunctionContext* ctx, const LargeIntVal& src, LargeIntVal* dst) {
- if (src.is_null) {
- return;
- }
-
- if (dst->is_null) {
- dst->is_null = false;
- dst->val = 0;
- }
-
- dst->val += src.val;
-}
-
template <typename T>
void AggregateFunctions::min_init(FunctionContext* ctx, T* dst) {
auto val = AnyValUtil::max_val<T>(ctx);
@@ -2395,6 +2379,8 @@ template void AggregateFunctions::sum<IntVal, BigIntVal>(FunctionContext*, const
BigIntVal* dst);
template void AggregateFunctions::sum<BigIntVal, BigIntVal>(FunctionContext*, const BigIntVal& src,
BigIntVal* dst);
+template void AggregateFunctions::sum<LargeIntVal, LargeIntVal>(FunctionContext*, const LargeIntVal& src,
+ LargeIntVal* dst);
template void AggregateFunctions::sum<FloatVal, DoubleVal>(FunctionContext*, const FloatVal& src,
DoubleVal* dst);
template void AggregateFunctions::sum<DoubleVal, DoubleVal>(FunctionContext*, const DoubleVal& src,
diff --git a/be/src/exprs/math_functions.cpp b/be/src/exprs/math_functions.cpp
index 787887d..00cf3bd 100644
--- a/be/src/exprs/math_functions.cpp
+++ b/be/src/exprs/math_functions.cpp
@@ -178,11 +178,11 @@ ONE_ARG_MATH_FN(ln, DoubleVal, DoubleVal, std::log);
ONE_ARG_MATH_FN(log10, DoubleVal, DoubleVal, std::log10);
ONE_ARG_MATH_FN(exp, DoubleVal, DoubleVal, std::exp);
-FloatVal MathFunctions::sign(FunctionContext* ctx, const DoubleVal& v) {
+TinyIntVal MathFunctions::sign(FunctionContext* ctx, const DoubleVal& v) {
if (v.is_null) {
- return FloatVal::null();
+ return TinyIntVal::null();
}
- return FloatVal((v.val > 0) ? 1.0f : ((v.val < 0) ? -1.0f : 0.0f));
+ return TinyIntVal((v.val > 0) ? 1 : ((v.val < 0) ? -1 : 0));
}
DoubleVal MathFunctions::radians(FunctionContext* ctx, const DoubleVal& v) {
diff --git a/be/src/exprs/math_functions.h b/be/src/exprs/math_functions.h
index 5d1258e..15d8749 100644
--- a/be/src/exprs/math_functions.h
+++ b/be/src/exprs/math_functions.h
@@ -50,7 +50,7 @@ public:
static doris_udf::IntVal abs(doris_udf::FunctionContext*, const doris_udf::SmallIntVal&);
static doris_udf::SmallIntVal abs(doris_udf::FunctionContext*, const doris_udf::TinyIntVal&);
- static doris_udf::FloatVal sign(doris_udf::FunctionContext* ctx, const doris_udf::DoubleVal& v);
+ static doris_udf::TinyIntVal sign(doris_udf::FunctionContext* ctx, const doris_udf::DoubleVal& v);
static doris_udf::DoubleVal sin(doris_udf::FunctionContext*, const doris_udf::DoubleVal&);
static doris_udf::DoubleVal asin(doris_udf::FunctionContext*, const doris_udf::DoubleVal&);
diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp
index 29830f2..21a3e84 100644
--- a/be/src/exprs/runtime_filter.cpp
+++ b/be/src/exprs/runtime_filter.cpp
@@ -362,6 +362,35 @@ public:
break;
}
}
+ void insert(const StringRef& value) {
+ switch (_column_return_type) {
+ case TYPE_DATE:
+ case TYPE_DATETIME: {
+ // DateTime->DateTimeValue
+ vectorized::DateTime date_time =*reinterpret_cast<const vectorized::DateTime*>(value.data);
+ vectorized::VecDateTimeValue vec_date_time_value = binary_cast<vectorized::Int64, vectorized::VecDateTimeValue>(date_time);
+ doris::DateTimeValue date_time_value;
+ vec_date_time_value.convert_vec_dt_to_dt(&date_time_value);
+ insert(reinterpret_cast<const void*>(&date_time_value));
+ break;
+ }
+
+ case TYPE_CHAR:
+ case TYPE_VARCHAR:
+ case TYPE_HLL:
+ case TYPE_OBJECT:
+ case TYPE_STRING: {
+ // StringRef->StringValue
+ StringValue data = StringValue(const_cast<char*>(value.data), value.size);
+ insert(reinterpret_cast<const void*>(&data));
+ break;
+ }
+
+ default:
+ insert(reinterpret_cast<const void*>(value.data));
+ break;
+ }
+ }
template <class T>
Status get_push_context(T* container, RuntimeState* state, ExprContext* prob_expr) {
@@ -613,6 +642,11 @@ void IRuntimeFilter::insert(const void* data) {
_wrapper->insert(data);
}
+void IRuntimeFilter::insert(const StringRef& value) {
+ DCHECK(is_producer());
+ _wrapper->insert(value);
+}
+
Status IRuntimeFilter::publish() {
DCHECK(is_producer());
if (_has_local_target) {
diff --git a/be/src/exprs/runtime_filter.h b/be/src/exprs/runtime_filter.h
index 809c132..f16b1d6 100644
--- a/be/src/exprs/runtime_filter.h
+++ b/be/src/exprs/runtime_filter.h
@@ -121,6 +121,7 @@ public:
// insert data to build filter
// only used for producer
void insert(const void* data);
+ void insert(const StringRef& data);
// publish filter
// push filter to remote node or push down it to scan_node
diff --git a/be/src/exprs/runtime_filter_slots.h b/be/src/exprs/runtime_filter_slots.h
index 7f0957a..3ac0ca6 100644
--- a/be/src/exprs/runtime_filter_slots.h
+++ b/be/src/exprs/runtime_filter_slots.h
@@ -20,9 +20,9 @@
#include "exprs/runtime_filter.h"
#include "runtime/runtime_filter_mgr.h"
#include "runtime/runtime_state.h"
+#include "vec/exprs/vexpr.h"
namespace doris {
-
// this class used in a hash join node
// Provide a unified interface for other classes
template <typename ExprCtxType>
@@ -96,6 +96,40 @@ public:
}
}
}
+ void insert(std::unordered_map<const vectorized::Block*, std::vector<int>>& datas) {
+ for (int i = 0; i < _build_expr_context.size(); ++i) {
+ auto iter = _runtime_filters.find(i);
+ if (iter == _runtime_filters.end()) continue;
+
+ int result_column_id = _build_expr_context[i]->get_last_result_column_id();
+ for (auto it : datas) {
+ auto& column = it.first->get_by_position(result_column_id).column;
+
+ if (auto* nullable =
+ vectorized::check_and_get_column<vectorized::ColumnNullable>(*column)) {
+ auto& column_nested = nullable->get_nested_column();
+ auto& column_nullmap = nullable->get_null_map_column();
+ for (int row_num : it.second) {
+ if (column_nullmap.get_bool(row_num)) {
+ continue;
+ }
+ const auto& ref_data = column_nested.get_data_at(row_num);
+ for (auto filter : iter->second) {
+ filter->insert(ref_data);
+ }
+ }
+
+ } else {
+ for (int row_num : it.second) {
+ const auto& ref_data = column->get_data_at(row_num);
+ for (auto filter : iter->second) {
+ filter->insert(ref_data);
+ }
+ }
+ }
+ }
+ }
+ }
// should call this method after insert
void ready_for_publish() {
@@ -133,5 +167,5 @@ private:
};
using RuntimeFilterSlots = RuntimeFilterSlotsBase<ExprContext>;
-
+using VRuntimeFilterSlots = RuntimeFilterSlotsBase<vectorized::VExprContext>;
} // namespace doris
diff --git a/be/src/exprs/v_string_functions.h b/be/src/exprs/v_string_functions.h
new file mode 100644
index 0000000..3fd9845
--- /dev/null
+++ b/be/src/exprs/v_string_functions.h
@@ -0,0 +1,219 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#ifndef BE_V_STRING_FUNCTIONS_H
+#define BE_V_STRING_FUNCTIONS_H
+
+#include <stdint.h>
+#include <unistd.h>
+#include "runtime/string_value.hpp"
+
+#ifdef __SSE2__
+#include <emmintrin.h>
+#endif
+
+namespace doris {
+class VStringFunctions {
+public:
+#ifdef __SSE2__
+ /// n equals to 16 chars length
+ static constexpr auto REGISTER_SIZE = sizeof(__m128i);
+#endif
+public:
+ static StringVal rtrim(const StringVal& str) {
+ if (str.is_null || str.len == 0) {
+ return str;
+ }
+ auto begin = 0;
+ auto end = str.len - 1;
+#ifdef __SSE2__
+ char blank = ' ';
+ const auto pattern = _mm_set1_epi8(blank);
+ while (end - begin + 1 >= REGISTER_SIZE) {
+ const auto v_haystack = _mm_loadu_si128(reinterpret_cast<const __m128i *>(str.ptr + end + 1 - REGISTER_SIZE));
+ const auto v_against_pattern = _mm_cmpeq_epi8(v_haystack, pattern);
+ const auto mask = _mm_movemask_epi8(v_against_pattern);
+ int offset = __builtin_clz(~(mask << REGISTER_SIZE));
+ /// means not found
+ if (offset == 0)
+ {
+ return StringVal(str.ptr + begin, end - begin + 1);
+ } else {
+ end -= offset;
+ }
+ }
+#endif
+ while (end >= begin && str.ptr[end] == ' ') {
+ --end;
+ }
+ if (end < 0) {
+ return StringVal("");
+ }
+ return StringVal(str.ptr + begin, end - begin + 1);
+ }
+
+ static StringVal ltrim(const StringVal& str) {
+ if (str.is_null || str.len == 0) {
+ return str;
+ }
+ auto begin = 0;
+ auto end = str.len - 1;
+#ifdef __SSE2__
+ char blank = ' ';
+ const auto pattern = _mm_set1_epi8(blank);
+ while (end - begin + 1 >= REGISTER_SIZE) {
+ const auto v_haystack = _mm_loadu_si128(reinterpret_cast<const __m128i *>(str.ptr + begin));
+ const auto v_against_pattern = _mm_cmpeq_epi8(v_haystack, pattern);
+ const auto mask = _mm_movemask_epi8(v_against_pattern);
+ const auto offset = __builtin_ctz(mask ^ 0xffff);
+ /// means not found
+ if (offset == 0)
+ {
+ return StringVal(str.ptr + begin, end - begin + 1);
+ } else if (offset > REGISTER_SIZE) {
+ begin += REGISTER_SIZE;
+ } else {
+ begin += offset;
+ return StringVal(str.ptr + begin, end - begin + 1);
+ }
+ }
+#endif
+ while (begin <= end && str.ptr[begin] == ' ') {
+ ++begin;
+ }
+ return StringVal(str.ptr + begin, end - begin + 1);
+ }
+
+ static StringVal trim(const StringVal& str) {
+ if (str.is_null || str.len == 0) {
+ return str;
+ }
+ return rtrim(ltrim(str));
+ }
+
+ static bool is_ascii(StringVal str) {
+ #ifdef __SSE2__
+ size_t i = 0;
+ __m128i binary_code = _mm_setzero_si128();
+ if (str.len >= REGISTER_SIZE) {
+ for (; i <= str.len - REGISTER_SIZE; i += REGISTER_SIZE) {
+ __m128i chars = _mm_loadu_si128((const __m128i*)(str.ptr + i));
+ binary_code = _mm_or_si128(binary_code, chars);
+ }
+ }
+ int mask = _mm_movemask_epi8(binary_code);
+
+ char or_code = 0;
+ for (; i < str.len; i++) {
+ or_code |= str.ptr[i];
+ }
+ mask |= (or_code & 0x80);
+
+ return !mask;
+ #else
+ char or_code = 0;
+ for (size_t i = 0; i < str.len; i++) {
+ or_code |= str.ptr[i];
+ }
+ return !(or_code & 0x80);
+ #endif
+ }
+
+ static void reverse(const StringVal& str, StringVal dst) {
+ if (str.is_null) {
+ dst.ptr = NULL;
+ return;
+ }
+ const bool is_ascii = VStringFunctions::is_ascii(str);
+ if (is_ascii) {
+ int64_t begin = 0;
+ int64_t end = str.len;
+ int64_t result_end = dst.len;
+ #if defined(__SSE2__)
+ const auto shuffle_array = _mm_set_epi64((__m64)0x00'01'02'03'04'05'06'07ull, (__m64)0x08'09'0a'0b'0c'0d'0e'0full);
+ for (; (begin + REGISTER_SIZE) < end; begin += REGISTER_SIZE) {
+ result_end -= REGISTER_SIZE;
+ _mm_storeu_si128((__m128i*)(dst.ptr + result_end),
+ _mm_shuffle_epi8(_mm_loadu_si128((__m128i*)(str.ptr + begin)), shuffle_array));
+ }
+ #endif
+ for (; begin < end; ++begin) {
+ --result_end;
+ dst.ptr[result_end] = str.ptr[begin];
+ }
+ } else {
+ for (size_t i = 0, char_size = 0; i < str.len; i += char_size) {
+ char_size = get_utf8_byte_length((unsigned)(str.ptr)[i]);
+ std::copy(str.ptr + i, str.ptr + i + char_size, dst.ptr + str.len - i - char_size);
+ }
+ }
+ }
+
+ static size_t get_utf8_byte_length(unsigned char byte) {
+ size_t char_size = 0;
+ if (byte >= 0xFC) {
+ char_size = 6;
+ } else if (byte >= 0xF8) {
+ char_size = 5;
+ } else if (byte >= 0xF0) {
+ char_size = 4;
+ } else if (byte >= 0xE0) {
+ char_size = 3;
+ } else if (byte >= 0xC0) {
+ char_size = 2;
+ } else {
+ char_size = 1;
+ }
+ return char_size;
+ }
+
+ static void hex_encode(const unsigned char* src_str, size_t length, char* dst_str) {
+ static constexpr auto hex_table = "0123456789ABCDEF";
+ auto src_str_end = src_str + length;
+
+#if defined(__SSE2__)
+ constexpr auto step = sizeof(uint64);
+ if (src_str + step < src_str_end) {
+ const auto hex_map = _mm_loadu_si128(reinterpret_cast<const __m128i *>(hex_table));
+ const auto mask_map = _mm_set1_epi8(0x0F);
+
+ do {
+ auto data = _mm_loadu_si64(src_str);
+ auto hex_loc = _mm_and_si128(_mm_unpacklo_epi8(_mm_srli_epi64(data, 4), data), mask_map);
+ _mm_storeu_si128(reinterpret_cast<__m128i *>(dst_str), _mm_shuffle_epi8(hex_map, hex_loc));
+
+ src_str += step;
+ dst_str += step * 2;
+ } while (src_str + step < src_str_end);
+ }
+#endif
+ char res[2];
+ // hex(str) str length is n, result must be 2 * n length
+ for (; src_str < src_str_end; src_str += 1, dst_str += 2) {
+ // low 4 bits
+ *(res + 1) = hex_table[src_str[0] & 0x0F];
+ // high 4 bits
+ *res = hex_table[(src_str[0] >> 4)];
+ std::copy(res, res + 2, dst_str);
+ }
+ }
+};
+}
+
+#endif //BE_V_STRING_FUNCTIONS_H
\ No newline at end of file
diff --git a/be/src/olap/block_column_predicate.cpp b/be/src/olap/block_column_predicate.cpp
index f460be5..725ebda 100644
--- a/be/src/olap/block_column_predicate.cpp
+++ b/be/src/olap/block_column_predicate.cpp
@@ -39,6 +39,30 @@ void SingleColumnBlockPredicate::evaluate_or(RowBlockV2 *block, uint16_t selecte
_predicate->evaluate_or(&column_block, block->selection_vector(), selected_size, flags);
}
+void SingleColumnBlockPredicate::evaluate(vectorized::MutableColumns& block, uint16_t* sel, uint16_t* selected_size) const {
+ auto column_id = _predicate->column_id();
+ auto& column = block[column_id];
+ _predicate->evaluate(*column, sel, selected_size);
+}
+
+void SingleColumnBlockPredicate::evaluate_and(vectorized::MutableColumns& block, uint16_t* sel, uint16_t selected_size, bool* flags) const {
+ auto column_id = _predicate->column_id();
+ auto& column = block[column_id];
+ _predicate->evaluate_and(*column, sel, selected_size, flags);
+}
+
+void SingleColumnBlockPredicate::evaluate_or(vectorized::MutableColumns& block, uint16_t* sel, uint16_t selected_size, bool* flags) const {
+ auto column_id = _predicate->column_id();
+ auto& column = block[column_id];
+ _predicate->evaluate_or(*column, sel, selected_size, flags);
+}
+
+void SingleColumnBlockPredicate::evaluate_vec(vectorized::MutableColumns& block, uint16_t size, bool* flags) const {
+ auto column_id = _predicate->column_id();
+ auto& column = block[column_id];
+ _predicate->evaluate_vec(*column, size, flags);
+}
+
void OrBlockColumnPredicate::evaluate(RowBlockV2* block, uint16_t* selected_size) const {
if (num_of_column_predicate() == 1) {
_block_column_predicate_vec[0]->evaluate(block, selected_size);
@@ -60,12 +84,39 @@ void OrBlockColumnPredicate::evaluate(RowBlockV2* block, uint16_t* selected_size
}
}
+void OrBlockColumnPredicate::evaluate(vectorized::MutableColumns& block, uint16_t* sel, uint16_t* selected_size) const {
+ if (num_of_column_predicate() == 1) {
+ _block_column_predicate_vec[0]->evaluate(block, sel, selected_size);
+ } else {
+ bool ret_flags[*selected_size];
+ memset(ret_flags, false, *selected_size);
+ for (int i = 0; i < num_of_column_predicate(); ++i) {
+ auto column_predicate = _block_column_predicate_vec[i];
+ column_predicate->evaluate_or(block, sel, *selected_size, ret_flags);
+ }
+
+ uint16_t new_size = 0;
+ for (int i = 0; i < *selected_size; ++i) {
+ if (ret_flags[i]) {
+ sel[new_size++] = sel[i];
+ }
+ }
+ *selected_size = new_size;
+ }
+}
+
void OrBlockColumnPredicate::evaluate_or(RowBlockV2 *block, uint16_t selected_size, bool* flags) const {
for (auto block_column_predicate : _block_column_predicate_vec) {
block_column_predicate->evaluate_or(block, selected_size, flags);
}
}
+void OrBlockColumnPredicate::evaluate_or(vectorized::MutableColumns& block, uint16_t* sel, uint16_t selected_size, bool* flags) const {
+ for (auto block_column_predicate : _block_column_predicate_vec) {
+ block_column_predicate->evaluate_or(block, sel, selected_size, flags);
+ }
+}
+
void OrBlockColumnPredicate::evaluate_and(RowBlockV2 *block, uint16_t selected_size, bool* flags) const {
if (num_of_column_predicate() == 1) {
_block_column_predicate_vec[0]->evaluate_and(block, selected_size, flags);
@@ -83,18 +134,47 @@ void OrBlockColumnPredicate::evaluate_and(RowBlockV2 *block, uint16_t selected_s
}
}
+void OrBlockColumnPredicate::evaluate_and(vectorized::MutableColumns& block, uint16_t* sel, uint16_t selected_size, bool* flags) const {
+ if (num_of_column_predicate() == 1) {
+ _block_column_predicate_vec[0]->evaluate_and(block, sel, selected_size, flags);
+ } else {
+ bool ret_flags[selected_size];
+ memset(ret_flags, false, selected_size);
+ for (int i = 0; i < num_of_column_predicate(); ++i) {
+ auto column_predicate = _block_column_predicate_vec[i];
+ column_predicate->evaluate_or(block, sel, selected_size, ret_flags);
+ }
+
+ for (int i = 0; i < selected_size; ++i) {
+ flags[i] &= ret_flags[i];
+ }
+ }
+}
+
void AndBlockColumnPredicate::evaluate(RowBlockV2* block, uint16_t* selected_size) const {
for (auto block_column_predicate : _block_column_predicate_vec) {
block_column_predicate->evaluate(block, selected_size);
}
}
+void AndBlockColumnPredicate::evaluate(vectorized::MutableColumns& block, uint16_t* sel, uint16_t* selected_size) const {
+ for (auto block_column_predicate : _block_column_predicate_vec) {
+ block_column_predicate->evaluate(block, sel, selected_size);
+ }
+}
+
void AndBlockColumnPredicate::evaluate_and(RowBlockV2 *block, uint16_t selected_size, bool* flags) const {
for (auto block_column_predicate : _block_column_predicate_vec) {
block_column_predicate->evaluate_and(block, selected_size, flags);
}
}
+void AndBlockColumnPredicate::evaluate_and(vectorized::MutableColumns& block, uint16_t* sel, uint16_t selected_size, bool* flags) const {
+ for (auto block_column_predicate : _block_column_predicate_vec) {
+ block_column_predicate->evaluate_and(block, sel, selected_size, flags);
+ }
+}
+
void AndBlockColumnPredicate::evaluate_or(RowBlockV2 *block, uint16_t selected_size, bool* flags) const {
if (num_of_column_predicate() == 1) {
_block_column_predicate_vec[0]->evaluate_or(block, selected_size, flags);
@@ -113,4 +193,38 @@ void AndBlockColumnPredicate::evaluate_or(RowBlockV2 *block, uint16_t selected_s
}
}
+void AndBlockColumnPredicate::evaluate_or(vectorized::MutableColumns& block, uint16_t* sel, uint16_t selected_size, bool* flags) const {
+ if (num_of_column_predicate() == 1) {
+ _block_column_predicate_vec[0]->evaluate_or(block, sel, selected_size, flags);
+ } else {
+ bool new_flags[selected_size];
+ memset(new_flags, true, selected_size);
+
+ for (auto block_column_predicate : _block_column_predicate_vec) {
+ block_column_predicate->evaluate_and(block, sel, selected_size, new_flags);
+ }
+
+ for (uint16_t i = 0; i < selected_size; i++) {
+ flags[i] |= new_flags[i];
+ }
+ }
+}
+
+// todo(wb) Can the 'and' of multiple bitmaps be vectorized?
+void AndBlockColumnPredicate::evaluate_vec(vectorized::MutableColumns& block, uint16_t size, bool* flags) const {
+ if (num_of_column_predicate() == 1) {
+ _block_column_predicate_vec[0]->evaluate_vec(block, size, flags);
+ } else {
+ bool new_flags[size];
+ for (auto block_column_predicate : _block_column_predicate_vec) {
+ memset(new_flags, true, size);
+ block_column_predicate->evaluate_vec(block, size, new_flags);
+
+ for (uint16_t j = 0; j < size; j++) {
+ flags[j] &= new_flags[j] ;
+ }
+ }
+ }
+}
+
} // namespace doris
diff --git a/be/src/olap/block_column_predicate.h b/be/src/olap/block_column_predicate.h
index 4c06486..ef2a4ca 100644
--- a/be/src/olap/block_column_predicate.h
+++ b/be/src/olap/block_column_predicate.h
@@ -44,6 +44,12 @@ public:
virtual void evaluate_or(RowBlockV2* block, uint16_t selected_size, bool* flags) const = 0;
virtual void get_all_column_ids(std::set<ColumnId>& column_id_set) const = 0;
+
+ virtual void evaluate(vectorized::MutableColumns& block, uint16_t* sel, uint16_t* selected_size) const {};
+ virtual void evaluate_and(vectorized::MutableColumns& block, uint16_t* sel, uint16_t selected_size, bool* flags) const {};
+ virtual void evaluate_or(vectorized::MutableColumns& block, uint16_t* sel, uint16_t selected_size, bool* flags) const {};
+
+ virtual void evaluate_vec(vectorized::MutableColumns& block, uint16_t size, bool* flags) const {};
};
class SingleColumnBlockPredicate : public BlockColumnPredicate {
@@ -57,6 +63,13 @@ public:
void get_all_column_ids(std::set<ColumnId>& column_id_set) const override {
column_id_set.insert(_predicate->column_id());
};
+
+ void evaluate(vectorized::MutableColumns& block, uint16_t* sel, uint16_t* selected_size) const override;
+ void evaluate_and(vectorized::MutableColumns& block, uint16_t* sel, uint16_t selected_size, bool* flags) const override;
+ void evaluate_or(vectorized::MutableColumns& block, uint16_t* sel, uint16_t selected_size, bool* flags) const override;
+
+ void evaluate_vec(vectorized::MutableColumns& block, uint16_t size, bool* flags) const override;
+
private:
const ColumnPredicate* _predicate;
};
@@ -98,6 +111,12 @@ public:
// 2.Do AND SEMANTICS in flags use 1 result to get proper select flags
void evaluate_and(RowBlockV2* block, uint16_t selected_size, bool* flags) const override;
void evaluate_or(RowBlockV2* block, uint16_t selected_size, bool* flags) const override;
+
+ void evaluate(vectorized::MutableColumns& block, uint16_t* sel, uint16_t* selected_size) const override;
+ void evaluate_and(vectorized::MutableColumns& block, uint16_t* sel, uint16_t selected_size, bool* flags) const override;
+ void evaluate_or(vectorized::MutableColumns& block, uint16_t* sel, uint16_t selected_size, bool* flags) const override;
+
+ // note(wb) we didnt't impelment evaluate_vec method here, because storage layer only support AND predicate now;
};
class AndBlockColumnPredicate : public MutilColumnBlockPredicate {
@@ -109,6 +128,13 @@ public:
// 1.AndBlockColumnPredicate need evaluate all child BlockColumnPredicate AND SEMANTICS inside first
// 2.Evaluate OR SEMANTICS in flags use 1 result to get proper select flags
void evaluate_or(RowBlockV2* block, uint16_t selected_size, bool* flags) const override;
+
+ void evaluate(vectorized::MutableColumns& block, uint16_t* sel, uint16_t* selected_size) const override;
+ void evaluate_and(vectorized::MutableColumns& block, uint16_t* sel, uint16_t selected_size, bool* flags) const override;
+ void evaluate_or(vectorized::MutableColumns& block, uint16_t* sel, uint16_t selected_size, bool* flags) const override;
+
+ void evaluate_vec(vectorized::MutableColumns& block, uint16_t size, bool* flags) const override;
+
};
} //namespace doris
diff --git a/be/src/olap/collect_iterator.cpp b/be/src/olap/collect_iterator.cpp
index 9ef3345..797b033 100644
--- a/be/src/olap/collect_iterator.cpp
+++ b/be/src/olap/collect_iterator.cpp
@@ -32,7 +32,7 @@ CollectIterator::~CollectIterator() = default;
void CollectIterator::init(Reader* reader) {
_reader = reader;
// when aggregate is enabled or key_type is DUP_KEYS, we don't merge
- // multiple data to aggregate for performance in user fetch
+ // multiple data to aggregate for better performance
if (_reader->_reader_type == READER_QUERY &&
(_reader->_aggregation || _reader->_tablet->keys_type() == KeysType::DUP_KEYS)) {
_merge = false;
diff --git a/be/src/olap/column_predicate.h b/be/src/olap/column_predicate.h
index 21b115e..10b8a91 100644
--- a/be/src/olap/column_predicate.h
+++ b/be/src/olap/column_predicate.h
@@ -23,6 +23,7 @@
#include "olap/column_block.h"
#include "olap/rowset/segment_v2/bitmap_index_reader.h"
#include "olap/selection_vector.h"
+#include "vec/columns/column.h"
using namespace doris::segment_v2;
@@ -54,6 +55,18 @@ public:
const std::vector<BitmapIndexIterator*>& iterators, uint32_t num_rows,
roaring::Roaring* roaring) const = 0;
+ // evaluate predicate on IColumn
+ // a short circuit eval way
+ virtual void evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) const {};
+ virtual void evaluate_and(vectorized::IColumn& column, uint16_t* sel, uint16_t size,
+ bool* flags) const {};
+ virtual void evaluate_or(vectorized::IColumn& column, uint16_t* sel, uint16_t size,
+ bool* flags) const {};
+
+ // used to evaluate pre read column in lazy matertialization
+ // now only support integer/float
+ // a vectorized eval way
+ virtual void evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) const {};
uint32_t column_id() const { return _column_id; }
protected:
diff --git a/be/src/olap/comparison_predicate.cpp b/be/src/olap/comparison_predicate.cpp
index 7dbe7cb..598e7f3 100644
--- a/be/src/olap/comparison_predicate.cpp
+++ b/be/src/olap/comparison_predicate.cpp
@@ -21,6 +21,9 @@
#include "olap/schema.h"
#include "runtime/string_value.hpp"
#include "runtime/vectorized_row_batch.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_vector.h"
+#include "vec/columns/predicate_column.h"
namespace doris {
@@ -142,6 +145,75 @@ COMPARISON_PRED_COLUMN_BLOCK_EVALUATE(LessEqualPredicate, <=)
COMPARISON_PRED_COLUMN_BLOCK_EVALUATE(GreaterPredicate, >)
COMPARISON_PRED_COLUMN_BLOCK_EVALUATE(GreaterEqualPredicate, >=)
+#define COMPARISON_PRED_COLUMN_EVALUATE(CLASS, OP) \
+ template <class type> \
+ void CLASS<type>::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) const { \
+ uint16_t new_size = 0; \
+ if (column.is_nullable()) { \
+ auto* nullable_column = vectorized::check_and_get_column<vectorized::ColumnNullable>(column);\
+ auto& null_bitmap = reinterpret_cast<const vectorized::ColumnVector<uint8_t>&>(*(nullable_column->get_null_map_column_ptr())).get_data(); \
+ auto* nest_column_vector = vectorized::check_and_get_column<vectorized::PredicateColumnType<type>>(nullable_column->get_nested_column());\
+ auto& data_array = nest_column_vector->get_data(); \
+ for (uint16_t i = 0; i < *size; i++) { \
+ uint16_t idx = sel[i]; \
+ sel[new_size] = idx; \
+ const type& cell_value = reinterpret_cast<const type&>(data_array[idx]); \
+ bool ret = !null_bitmap[idx] && (cell_value OP _value); \
+ new_size += _opposite ? !ret : ret; \
+ } \
+ *size = new_size; \
+ } else {\
+ auto& pred_column_ref = reinterpret_cast<vectorized::PredicateColumnType<type>&>(column);\
+ auto& data_array = pred_column_ref.get_data(); \
+ for (uint16_t i = 0; i < *size; i++) { \
+ uint16_t idx = sel[i]; \
+ sel[new_size] = idx; \
+ const type& cell_value = reinterpret_cast<const type&>(data_array[idx]); \
+ auto ret = cell_value OP _value; \
+ new_size += _opposite ? !ret : ret; \
+ } \
+ *size = new_size; \
+ }\
+ }
+
+COMPARISON_PRED_COLUMN_EVALUATE(EqualPredicate, ==)
+COMPARISON_PRED_COLUMN_EVALUATE(NotEqualPredicate, !=)
+COMPARISON_PRED_COLUMN_EVALUATE(LessPredicate, <)
+COMPARISON_PRED_COLUMN_EVALUATE(LessEqualPredicate, <=)
+COMPARISON_PRED_COLUMN_EVALUATE(GreaterPredicate, >)
+COMPARISON_PRED_COLUMN_EVALUATE(GreaterEqualPredicate, >=)
+
+#define COMPARISON_PRED_COLUMN_EVALUATE_VEC(CLASS, OP) \
+ template <class type> \
+ void CLASS<type>::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) const { \
+ if (column.is_nullable()) { \
+ auto* nullable_column = vectorized::check_and_get_column<vectorized::ColumnNullable>(column); \
+ auto& data_array = reinterpret_cast<const vectorized::ColumnVector<type>&>(nullable_column->get_nested_column()).get_data(); \
+ auto& null_bitmap = reinterpret_cast<const vectorized::ColumnVector<uint8_t>&>(*(nullable_column->get_null_map_column_ptr())).get_data(); \
+ for (uint16_t i = 0; i < size; i++) { \
+ flags[i] = (data_array[i] OP _value) && (!null_bitmap[i]); \
+ } \
+ } else { \
+ auto& predicate_column = reinterpret_cast<vectorized::PredicateColumnType<type>&>(column); \
+ auto& data_array = predicate_column.get_data(); \
+ for (uint16_t i = 0; i < size; i++) { \
+ flags[i] = data_array[i] OP _value; \
+ } \
+ } \
+ if (_opposite) { \
+ for (uint16_t i = 0; i < size; i++) { \
+ flags[i] = !flags[i]; \
+ } \
+ } \
+ }
+
+COMPARISON_PRED_COLUMN_EVALUATE_VEC(EqualPredicate, ==)
+COMPARISON_PRED_COLUMN_EVALUATE_VEC(NotEqualPredicate, !=)
+COMPARISON_PRED_COLUMN_EVALUATE_VEC(LessPredicate, <)
+COMPARISON_PRED_COLUMN_EVALUATE_VEC(LessEqualPredicate, <=)
+COMPARISON_PRED_COLUMN_EVALUATE_VEC(GreaterPredicate, >)
+COMPARISON_PRED_COLUMN_EVALUATE_VEC(GreaterEqualPredicate, >=)
+
#define COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_OR(CLASS, OP) \
template <class type> \
void CLASS<type>::evaluate_or(ColumnBlock* block, uint16_t* sel, uint16_t size, bool* flags) \
@@ -174,6 +246,20 @@ COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_OR(LessEqualPredicate, <=)
COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_OR(GreaterPredicate, >)
COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_OR(GreaterEqualPredicate, >=)
+// todo(wb) support it
+#define COMPARISON_PRED_COLUMN_EVALUATE_OR(CLASS, OP) \
+ template <class type> \
+ void CLASS<type>::evaluate_or(vectorized::IColumn& column, uint16_t* sel, uint16_t size, bool* flags) const { \
+ \
+ }
+
+COMPARISON_PRED_COLUMN_EVALUATE_OR(EqualPredicate, ==)
+COMPARISON_PRED_COLUMN_EVALUATE_OR(NotEqualPredicate, !=)
+COMPARISON_PRED_COLUMN_EVALUATE_OR(LessPredicate, <)
+COMPARISON_PRED_COLUMN_EVALUATE_OR(LessEqualPredicate, <=)
+COMPARISON_PRED_COLUMN_EVALUATE_OR(GreaterPredicate, >)
+COMPARISON_PRED_COLUMN_EVALUATE_OR(GreaterEqualPredicate, >=)
+
#define COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_AND(CLASS, OP) \
template <class type> \
void CLASS<type>::evaluate_and(ColumnBlock* block, uint16_t* sel, uint16_t size, bool* flags) \
@@ -206,6 +292,21 @@ COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_AND(LessEqualPredicate, <=)
COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_AND(GreaterPredicate, >)
COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_AND(GreaterEqualPredicate, >=)
+//todo(wb) support it
+#define COMPARISON_PRED_COLUMN_EVALUATE_AND(CLASS, OP) \
+ template <class type> \
+ void CLASS<type>::evaluate_and(vectorized::IColumn& column, uint16_t* sel, uint16_t size, bool* flags) const { \
+ \
+ \
+ }
+
+COMPARISON_PRED_COLUMN_EVALUATE_AND(EqualPredicate, ==)
+COMPARISON_PRED_COLUMN_EVALUATE_AND(NotEqualPredicate, !=)
+COMPARISON_PRED_COLUMN_EVALUATE_AND(LessPredicate, <)
+COMPARISON_PRED_COLUMN_EVALUATE_AND(LessEqualPredicate, <=)
+COMPARISON_PRED_COLUMN_EVALUATE_AND(GreaterPredicate, >)
+COMPARISON_PRED_COLUMN_EVALUATE_AND(GreaterEqualPredicate, >=)
+
#define BITMAP_COMPARE_EqualPredicate(s, exact_match, seeked_ordinal, iterator, bitmap, roaring) \
do { \
if (!s.is_not_found()) { \
@@ -440,4 +541,66 @@ COMPARISON_PRED_BITMAP_EVALUATE_DECLARATION(LessEqualPredicate)
COMPARISON_PRED_BITMAP_EVALUATE_DECLARATION(GreaterPredicate)
COMPARISON_PRED_BITMAP_EVALUATE_DECLARATION(GreaterEqualPredicate)
+#define COMPARISON_PRED_COLUMN_EVALUATE_DECLARATION(CLASS) \
+ template void CLASS<int8_t>::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) \
+ const; \
+ template void CLASS<int16_t>::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) \
+ const; \
+ template void CLASS<int32_t>::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) \
+ const; \
+ template void CLASS<int64_t>::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) \
+ const; \
+ template void CLASS<int128_t>::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) \
+ const; \
+ template void CLASS<float>::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) const; \
+ template void CLASS<double>::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) \
+ const; \
+ template void CLASS<decimal12_t>::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) \
+ const; \
+ template void CLASS<StringValue>::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) \
+ const; \
+ template void CLASS<uint24_t>::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) \
+ const; \
+ template void CLASS<uint64_t>::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) \
+ const; \
+ template void CLASS<bool>::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) const;
+
+COMPARISON_PRED_COLUMN_EVALUATE_DECLARATION(EqualPredicate)
+COMPARISON_PRED_COLUMN_EVALUATE_DECLARATION(NotEqualPredicate)
+COMPARISON_PRED_COLUMN_EVALUATE_DECLARATION(LessPredicate)
+COMPARISON_PRED_COLUMN_EVALUATE_DECLARATION(LessEqualPredicate)
+COMPARISON_PRED_COLUMN_EVALUATE_DECLARATION(GreaterPredicate)
+COMPARISON_PRED_COLUMN_EVALUATE_DECLARATION(GreaterEqualPredicate)
+
+#define COMPARISON_PRED_COLUMN_EVALUATE_VEC_DECLARATION(CLASS) \
+ template void CLASS<int8_t>::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) \
+ const; \
+ template void CLASS<int16_t>::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) \
+ const; \
+ template void CLASS<int32_t>::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) \
+ const; \
+ template void CLASS<int64_t>::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) \
+ const; \
+ template void CLASS<int128_t>::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) \
+ const; \
+ template void CLASS<float>::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) const; \
+ template void CLASS<double>::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) \
+ const; \
+ template void CLASS<decimal12_t>::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) \
+ const; \
+ template void CLASS<StringValue>::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) \
+ const; \
+ template void CLASS<uint24_t>::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) \
+ const; \
+ template void CLASS<uint64_t>::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) \
+ const; \
+ template void CLASS<bool>::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) const;
+
+COMPARISON_PRED_COLUMN_EVALUATE_VEC_DECLARATION(EqualPredicate)
+COMPARISON_PRED_COLUMN_EVALUATE_VEC_DECLARATION(NotEqualPredicate)
+COMPARISON_PRED_COLUMN_EVALUATE_VEC_DECLARATION(LessPredicate)
+COMPARISON_PRED_COLUMN_EVALUATE_VEC_DECLARATION(LessEqualPredicate)
+COMPARISON_PRED_COLUMN_EVALUATE_VEC_DECLARATION(GreaterPredicate)
+COMPARISON_PRED_COLUMN_EVALUATE_VEC_DECLARATION(GreaterEqualPredicate)
+
} //namespace doris
diff --git a/be/src/olap/comparison_predicate.h b/be/src/olap/comparison_predicate.h
index d0a4049..30fd9fd 100644
--- a/be/src/olap/comparison_predicate.h
+++ b/be/src/olap/comparison_predicate.h
@@ -40,6 +40,10 @@ class VectorizedRowBatch;
virtual Status evaluate(const Schema& schema, \
const std::vector<BitmapIndexIterator*>& iterators, \
uint32_t num_rows, roaring::Roaring* roaring) const override; \
+ void evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) const override; \
+ void evaluate_and(vectorized::IColumn& column, uint16_t* sel, uint16_t size, bool* flags) const override; \
+ void evaluate_or(vectorized::IColumn& column, uint16_t* sel, uint16_t size, bool* flags) const override; \
+ void evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) const override; \
\
private: \
type _value; \
diff --git a/be/src/olap/generic_iterators.cpp b/be/src/olap/generic_iterators.cpp
index adac5df..8a11108 100644
--- a/be/src/olap/generic_iterators.cpp
+++ b/be/src/olap/generic_iterators.cpp
@@ -116,6 +116,11 @@ public:
MergeIteratorContext(RowwiseIterator* iter, std::shared_ptr<MemTracker> parent)
: _iter(iter), _block(iter->schema(), 1024, std::move(parent)) {}
+ MergeIteratorContext(const MergeIteratorContext&) = delete;
+ MergeIteratorContext(MergeIteratorContext&&) = delete;
+ MergeIteratorContext& operator=(const MergeIteratorContext&) = delete;
+ MergeIteratorContext& operator=(MergeIteratorContext&&) = delete;
+
~MergeIteratorContext() {
delete _iter;
_iter = nullptr;
@@ -151,6 +156,7 @@ private:
private:
RowwiseIterator* _iter;
+
// used to store data load from iterator
RowBlockV2 _block;
@@ -181,10 +187,9 @@ Status MergeIteratorContext::advance() {
}
Status MergeIteratorContext::_load_next_block() {
- Status st;
do {
_block.clear();
- st = _iter->next_batch(&_block);
+ Status st = _iter->next_batch(&_block);
if (!st.ok()) {
_valid = false;
if (st.is_end_of_file()) {
@@ -202,27 +207,29 @@ Status MergeIteratorContext::_load_next_block() {
class MergeIterator : public RowwiseIterator {
public:
// MergeIterator takes the ownership of input iterators
- MergeIterator(std::list<RowwiseIterator*> iters, std::shared_ptr<MemTracker> parent, int sequence_id_idx)
- : _origin_iters(std::move(iters)), _sequence_id_idx(sequence_id_idx) {
+ MergeIterator(std::vector<RowwiseIterator*> iters, std::shared_ptr<MemTracker> parent, int sequence_id_idx)
+ : _origin_iters(std::move(iters)), _sequence_id_idx(sequence_id_idx), _merge_heap(MergeContextComparator(_sequence_id_idx)) {
// use for count the mem use of Block use in Merge
_mem_tracker = MemTracker::CreateTracker(-1, "MergeIterator", parent, false);
}
~MergeIterator() override {
- while (!_merge_heap->empty()) {
- auto ctx = _merge_heap->top();
- _merge_heap->pop();
+ while (!_merge_heap.empty()) {
+ auto ctx = _merge_heap.top();
+ _merge_heap.pop();
delete ctx;
}
}
+
Status init(const StorageReadOptions& opts) override;
+
Status next_batch(RowBlockV2* block) override;
const Schema& schema() const override { return *_schema; }
private:
// It will be released after '_merge_heap' has been built.
- std::list<RowwiseIterator*> _origin_iters;
+ std::vector<RowwiseIterator*> _origin_iters;
int _sequence_id_idx;
@@ -256,9 +263,12 @@ private:
int sequence_id_idx;
};
- using MergeHeap = std::priority_queue<MergeIteratorContext*, std::vector<MergeIteratorContext*>,
- MergeContextComparator>;
- std::unique_ptr<MergeHeap> _merge_heap;
+
+ using MergeHeap = std::priority_queue<MergeIteratorContext*,
+ std::vector<MergeIteratorContext*>,
+ MergeContextComparator>;
+
+ MergeHeap _merge_heap;
};
Status MergeIterator::init(const StorageReadOptions& opts) {
@@ -267,24 +277,24 @@ Status MergeIterator::init(const StorageReadOptions& opts) {
}
_schema.reset(new Schema((*(_origin_iters.begin()))->schema()));
- _merge_heap = std::make_unique<MergeHeap>(MergeContextComparator(_sequence_id_idx));
for (auto iter : _origin_iters) {
std::unique_ptr<MergeIteratorContext> ctx(new MergeIteratorContext(iter, _mem_tracker));
RETURN_IF_ERROR(ctx->init(opts));
if (!ctx->valid()) {
continue;
}
- _merge_heap->push(ctx.release());
+ _merge_heap.push(ctx.release());
}
+
_origin_iters.clear();
return Status::OK();
}
Status MergeIterator::next_batch(RowBlockV2* block) {
size_t row_idx = 0;
- for (; row_idx < block->capacity() && !_merge_heap->empty(); ++row_idx) {
- auto ctx = _merge_heap->top();
- _merge_heap->pop();
+ for (; row_idx < block->capacity() && !_merge_heap.empty(); ++row_idx) {
+ auto ctx = _merge_heap.top();
+ _merge_heap.pop();
RowBlockRow dst_row = block->row(row_idx);
// copy current row to block
@@ -292,7 +302,7 @@ Status MergeIterator::next_batch(RowBlockV2* block) {
RETURN_IF_ERROR(ctx->advance());
if (ctx->valid()) {
- _merge_heap->push(ctx);
+ _merge_heap.push(ctx);
} else {
// Release ctx earlier to reduce resource consumed
delete ctx;
@@ -313,17 +323,17 @@ public:
// Iterators' ownership it transfered to this class.
// This class will delete all iterators when destructs
// Client should not use iterators any more.
- UnionIterator(std::list<RowwiseIterator*> iters, std::shared_ptr<MemTracker> parent)
- : _origin_iters(std::move(iters)) {
+ UnionIterator(std::vector<RowwiseIterator*> &v, std::shared_ptr<MemTracker> parent)
+ : _origin_iters(v.begin(), v.end()) {
_mem_tracker = MemTracker::CreateTracker(-1, "UnionIterator", parent, false);
}
~UnionIterator() override {
- for (auto iter : _origin_iters) {
- delete iter;
- }
+ std::for_each(_origin_iters.begin(), _origin_iters.end(), std::default_delete<RowwiseIterator>());
}
+
Status init(const StorageReadOptions& opts) override;
+
Status next_batch(RowBlockV2* block) override;
const Schema& schema() const override { return *_schema; }
@@ -331,7 +341,7 @@ public:
private:
std::unique_ptr<Schema> _schema;
RowwiseIterator* _cur_iter = nullptr;
- std::list<RowwiseIterator*> _origin_iters;
+ std::deque<RowwiseIterator*> _origin_iters;
};
Status UnionIterator::init(const StorageReadOptions& opts) {
@@ -364,18 +374,18 @@ Status UnionIterator::next_batch(RowBlockV2* block) {
return Status::EndOfFile("End of UnionIterator");
}
-RowwiseIterator* new_merge_iterator(std::list<RowwiseIterator*> inputs, std::shared_ptr<MemTracker> parent, int sequence_id_idx) {
+RowwiseIterator* new_merge_iterator(std::vector<RowwiseIterator*> inputs, std::shared_ptr<MemTracker> parent, int sequence_id_idx) {
if (inputs.size() == 1) {
return *(inputs.begin());
}
return new MergeIterator(std::move(inputs), parent, sequence_id_idx);
}
-RowwiseIterator* new_union_iterator(std::list<RowwiseIterator*> inputs, std::shared_ptr<MemTracker> parent) {
+RowwiseIterator* new_union_iterator(std::vector<RowwiseIterator*>& inputs, std::shared_ptr<MemTracker> parent) {
if (inputs.size() == 1) {
return *(inputs.begin());
}
- return new UnionIterator(std::move(inputs), parent);
+ return new UnionIterator(inputs, parent);
}
RowwiseIterator* new_auto_increment_iterator(const Schema& schema, size_t num_rows) {
diff --git a/be/src/olap/generic_iterators.h b/be/src/olap/generic_iterators.h
index 1d3eccf..e8f4528 100644
--- a/be/src/olap/generic_iterators.h
+++ b/be/src/olap/generic_iterators.h
@@ -25,14 +25,14 @@ namespace doris {
//
// Inputs iterators' ownership is taken by created merge iterator. And client
// should delete returned iterator after usage.
-RowwiseIterator* new_merge_iterator(std::list<RowwiseIterator*> inputs, std::shared_ptr<MemTracker> parent, int sequence_id_idx);
+RowwiseIterator* new_merge_iterator(std::vector<RowwiseIterator*> inputs, std::shared_ptr<MemTracker> parent, int sequence_id_idx);
// Create a union iterator for input iterators. Union iterator will read
// input iterators one by one.
//
// Inputs iterators' ownership is taken by created union iterator. And client
// should delete returned iterator after usage.
-RowwiseIterator* new_union_iterator(std::list<RowwiseIterator*> inputs, std::shared_ptr<MemTracker> parent);
+RowwiseIterator* new_union_iterator(std::vector<RowwiseIterator*>& inputs, std::shared_ptr<MemTracker> parent);
// Create an auto increment iterator which returns num_rows data in format of schema.
// This class aims to be used in unit test.
diff --git a/be/src/olap/iterators.h b/be/src/olap/iterators.h
index d514dc0..4cdfc60 100644
--- a/be/src/olap/iterators.h
+++ b/be/src/olap/iterators.h
@@ -23,6 +23,7 @@
#include "olap/olap_common.h"
#include "olap/column_predicate.h"
#include "olap/block_column_predicate.h"
+#include "vec/core/block.h"
namespace doris {
@@ -80,6 +81,7 @@ public:
// REQUIRED (null is not allowed)
OlapReaderStatistics* stats = nullptr;
bool use_page_cache = false;
+ int block_row_max = 4096;
};
// Used to read data in RowBlockV2 one by one
@@ -99,7 +101,9 @@ public:
// into input batch with Status::OK() returned
// If there is no data to read, will return Status::EndOfFile.
// If other error happens, other error code will be returned.
- virtual Status next_batch(RowBlockV2* block) = 0;
+ virtual Status next_batch(RowBlockV2* block) { return Status::NotSupported("to be implemented"); }
+
+ virtual Status next_batch(vectorized::Block* block) { return Status::NotSupported("to be implemented"); }
// return schema for this Iterator
virtual const Schema& schema() const = 0;
diff --git a/be/src/olap/null_predicate.cpp b/be/src/olap/null_predicate.cpp
index cc88d26..da3eb29 100644
--- a/be/src/olap/null_predicate.cpp
+++ b/be/src/olap/null_predicate.cpp
@@ -20,6 +20,9 @@
#include "olap/field.h"
#include "runtime/string_value.hpp"
#include "runtime/vectorized_row_batch.h"
+#include "vec/columns/column_nullable.h"
+
+using namespace doris::vectorized;
namespace doris {
@@ -118,4 +121,44 @@ Status NullPredicate::evaluate(const Schema& schema,
return Status::OK();
}
+void NullPredicate::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) const {
+ uint16_t new_size = 0;
+ if (auto* nullable = check_and_get_column<ColumnNullable>(column)) {
+ auto& null_map = nullable->get_null_map_data();
+ for (uint16_t i = 0; i < *size; ++i) {
+ uint16_t idx = sel[i];
+ sel[new_size] = idx;
+ new_size += (null_map[idx] == _is_null);
+ }
+ *size = new_size;
+ } else {
+ if (_is_null) *size = 0;
+ }
+}
+
+void NullPredicate::evaluate_or(IColumn& column, uint16_t* sel, uint16_t size, bool* flags) const {
+ if (auto* nullable = check_and_get_column<ColumnNullable>(column)) {
+ auto& null_map = nullable->get_null_map_data();
+ for (uint16_t i = 0; i < size; ++i) {
+ if (flags[i]) continue;
+ uint16_t idx = sel[i];
+ flags[i] |= (null_map[idx] == _is_null);
+ }
+ } else {
+ if (!_is_null) memset(flags, true, size);
+ }
+}
+
+void NullPredicate::evaluate_and(IColumn& column, uint16_t* sel, uint16_t size, bool* flags) const {
+ if (auto* nullable = check_and_get_column<ColumnNullable>(column)) {
+ auto& null_map = nullable->get_null_map_data();
+ for (uint16_t i = 0; i < size; ++i) {
+ if (flags[i]) continue;
+ uint16_t idx = sel[i];
+ flags[i] &= (null_map[idx] == _is_null);
+ }
+ } else {
+ if (_is_null) memset(flags, false, size);
+ }
+}
} //namespace doris
diff --git a/be/src/olap/null_predicate.h b/be/src/olap/null_predicate.h
index e867d58..681e60b 100644
--- a/be/src/olap/null_predicate.h
+++ b/be/src/olap/null_predicate.h
@@ -43,6 +43,14 @@ public:
virtual Status evaluate(const Schema& schema, const vector<BitmapIndexIterator*>& iterators,
uint32_t num_rows, roaring::Roaring* roaring) const override;
+ void evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) const override;
+
+ void evaluate_or(vectorized::IColumn& column, uint16_t* sel, uint16_t size,
+ bool* flags) const override;
+
+ void evaluate_and(vectorized::IColumn& column, uint16_t* sel, uint16_t size,
+ bool* flags) const override;
+
private:
bool _is_null; //true for null, false for not null
};
diff --git a/be/src/olap/reader.cpp b/be/src/olap/reader.cpp
index 1e68469..335d4ca 100644
--- a/be/src/olap/reader.cpp
+++ b/be/src/olap/reader.cpp
@@ -41,6 +41,7 @@
#include "runtime/string_value.hpp"
#include "util/date_func.h"
#include "util/mem_util.hpp"
+#include "vec/olap/vcollect_iterator.h"
using std::nothrow;
using std::set;
@@ -57,8 +58,8 @@ void ReaderParams::check_validation() const {
std::string ReaderParams::to_string() const {
std::stringstream ss;
ss << "tablet=" << tablet->full_name() << " reader_type=" << reader_type
- << " aggregation=" << aggregation << " version=" << version << " start_key_include=" << start_key_include
- << " end_key_include=" << end_key_include;
+ << " aggregation=" << aggregation << " version=" << version
+ << " start_key_include=" << start_key_include << " end_key_include=" << end_key_include;
for (const auto& key : start_key) {
ss << " keys=" << key;
@@ -131,7 +132,7 @@ OLAPStatus Reader::init(const ReaderParams& read_params) {
<< ", version:" << read_params.version;
return res;
}
-
+
return OLAP_SUCCESS;
}
@@ -244,6 +245,7 @@ OLAPStatus Reader::_capture_rs_readers(const ReaderParams& read_params,
OLAPStatus Reader::_init_params(const ReaderParams& read_params) {
read_params.check_validation();
+ _direct_mode = read_params.direct_mode;
_aggregation = read_params.aggregation;
_need_agg_finalize = read_params.need_agg_finalize;
_reader_type = read_params.reader_type;
@@ -721,7 +723,7 @@ ColumnPredicate* Reader::_parse_to_predicate(const TCondition& condition, bool o
break;
}
case OLAP_FIELD_TYPE_VARCHAR:
- case OLAP_FIELD_TYPE_STRING:{
+ case OLAP_FIELD_TYPE_STRING: {
phmap::flat_hash_set<StringValue> values;
for (auto& cond_val : condition.condition_values) {
StringValue value;
@@ -828,7 +830,8 @@ void Reader::_init_load_bf_columns(const ReaderParams& read_params, Conditions*
return;
}
FieldType type = _tablet->tablet_schema().column(max_equal_index).type();
- if ((type != OLAP_FIELD_TYPE_VARCHAR && type != OLAP_FIELD_TYPE_STRING)|| max_equal_index + 1 > _tablet->num_short_key_columns()) {
+ if ((type != OLAP_FIELD_TYPE_VARCHAR && type != OLAP_FIELD_TYPE_STRING) ||
+ max_equal_index + 1 > _tablet->num_short_key_columns()) {
load_bf_columns->erase(max_equal_index);
}
}
diff --git a/be/src/olap/reader.h b/be/src/olap/reader.h
index 6fcbd93..fc96623 100644
--- a/be/src/olap/reader.h
+++ b/be/src/olap/reader.h
@@ -48,11 +48,17 @@ class RowBlock;
class CollectIterator;
class RuntimeState;
+namespace vectorized {
+class VCollectIterator;
+class Block;
+} // namespace vectorized
+
// Params for Reader,
// mainly include tablet, data version and fetch range.
struct ReaderParams {
TabletSharedPtr tablet;
ReaderType reader_type = READER_QUERY;
+ bool direct_mode = false;
bool aggregation = false;
bool need_agg_finalize = true;
// 1. when read column data page:
@@ -77,6 +83,9 @@ struct ReaderParams {
RuntimeProfile* profile = nullptr;
RuntimeState* runtime_state = nullptr;
+ // use only in vec unique key
+ std::vector<uint32_t>* origin_return_columns = nullptr;
+
void check_validation() const;
std::string to_string() const;
@@ -106,7 +115,17 @@ public:
// Return OLAP_SUCCESS and set `*eof` to true when no more rows can be read.
// Return others when unexpected error happens.
virtual OLAPStatus next_row_with_aggregation(RowCursor* row_cursor, MemPool* mem_pool,
- ObjectPool* agg_pool, bool* eof) = 0;
+ ObjectPool* agg_pool, bool* eof) = 0;
+
+ // Read next block with aggregation.
+ // Return OLAP_SUCCESS and set `*eof` to false when next block is read
+ // Return OLAP_SUCCESS and set `*eof` to true when no more rows can be read.
+ // Return others when unexpected error happens.
+ // TODO: Rethink here we still need mem_pool and agg_pool?
+ virtual OLAPStatus next_block_with_aggregation(vectorized::Block* block, MemPool* mem_pool,
+ ObjectPool* agg_pool, bool* eof) {
+ return OLAP_ERR_READER_INITIALIZE_ERROR;
+ }
uint64_t merged_rows() const { return _merged_rows; }
@@ -120,6 +139,7 @@ public:
protected:
friend class CollectIterator;
+ friend class vectorized::VCollectIterator;
friend class DeleteHandler;
OLAPStatus _init_params(const ReaderParams& read_params);
@@ -189,8 +209,8 @@ protected:
ReaderType _reader_type = READER_QUERY;
bool _next_delete_flag = false;
bool _filter_delete = false;
- bool _has_sequence_col = false;
int32_t _sequence_col_idx = -1;
+ bool _direct_mode = false;
std::unique_ptr<CollectIterator> _collect_iter;
std::vector<uint32_t> _key_cids;
diff --git a/be/src/olap/row_block2.cpp b/be/src/olap/row_block2.cpp
index 20ed4ae..26b58ca 100644
--- a/be/src/olap/row_block2.cpp
+++ b/be/src/olap/row_block2.cpp
@@ -23,6 +23,11 @@
#include "gutil/strings/substitute.h"
#include "olap/row_cursor.h"
#include "util/bitmap.h"
+#include "vec/columns/column_complex.h"
+#include "vec/columns/column_vector.h"
+#include "vec/core/block.h"
+#include "vec/core/types.h"
+#include "vec/runtime/vdatetime_value.h"
using strings::Substitute;
namespace doris {
@@ -90,6 +95,209 @@ Status RowBlockV2::convert_to_row_block(RowCursor* helper, RowBlock* dst) {
return Status::OK();
}
+void RowBlockV2::_copy_data_to_column(int cid, doris::vectorized::MutableColumnPtr& origin_column) {
+ auto* column = origin_column.get();
+ bool nullable_mark_array[_selected_size];
+
+ bool column_nullable = origin_column->is_nullable();
+ bool origin_nullable = _schema.column(cid)->is_nullable();
+ if (column_nullable) {
+ auto nullable_column = assert_cast<vectorized::ColumnNullable*>(origin_column.get());
+ auto& null_map = nullable_column->get_null_map_data();
+ column = nullable_column->get_nested_column_ptr().get();
+
+ if (origin_nullable) {
+ for (uint16_t i = 0; i < _selected_size; ++i) {
+ uint16_t row_idx = _selection_vector[i];
+ null_map.push_back(_column_vector_batches[cid]->is_null_at(row_idx));
+ nullable_mark_array[i] = null_map.back();
+ }
+ } else {
+ null_map.resize_fill(null_map.size() + _selected_size, 0);
+ memset(nullable_mark_array, false, _selected_size * sizeof(bool));
+ }
+ } else {
+ memset(nullable_mark_array, false, _selected_size * sizeof(bool));
+ }
+
+ auto insert_data_directly = [this, &nullable_mark_array](int cid, auto& column) {
+ for (uint16_t j = 0; j < _selected_size; ++j) {
+ if (!nullable_mark_array[j]) {
+ uint16_t row_idx = _selection_vector[j];
+ column->insert_data(
+ reinterpret_cast<const char*>(column_block(cid).cell_ptr(row_idx)), 0);
+ } else {
+ column->insert_default();
+ }
+ }
+ };
+
+ switch (_schema.column(cid)->type()) {
+ case OLAP_FIELD_TYPE_OBJECT: {
+ auto column_bitmap = assert_cast<vectorized::ColumnBitmap*>(column);
+ for (uint16_t j = 0; j < _selected_size; ++j) {
+ column_bitmap->insert_default();
+ if (!nullable_mark_array[j]) {
+ uint16_t row_idx = _selection_vector[j];
+ auto slice = reinterpret_cast<const Slice*>(column_block(cid).cell_ptr(row_idx));
+
+ BitmapValue* pvalue = &column_bitmap->get_element(column_bitmap->size() - 1);
+
+ if (slice->size != 0) {
+ BitmapValue value;
+ value.deserialize(slice->data);
+ *pvalue = std::move(value);
+ } else {
+ *pvalue = std::move(*reinterpret_cast<BitmapValue*>(slice->data));
+ }
+ }
+ }
+ break;
+ }
+ case OLAP_FIELD_TYPE_HLL:
+ case OLAP_FIELD_TYPE_MAP:
+ case OLAP_FIELD_TYPE_VARCHAR: {
+ auto column_string = assert_cast<vectorized::ColumnString*>(column);
+
+ for (uint16_t j = 0; j < _selected_size; ++j) {
+ if (!nullable_mark_array[j]) {
+ uint16_t row_idx = _selection_vector[j];
+ auto slice = reinterpret_cast<const Slice*>(column_block(cid).cell_ptr(row_idx));
+ column_string->insert_data(slice->data, slice->size);
+ } else {
+ column_string->insert_default();
+ }
+ }
+ break;
+ }
+ case OLAP_FIELD_TYPE_CHAR: {
+ auto column_string = assert_cast<vectorized::ColumnString*>(column);
+
+ for (uint16_t j = 0; j < _selected_size; ++j) {
+ if (!nullable_mark_array[j]) {
+ uint16_t row_idx = _selection_vector[j];
+ auto slice = reinterpret_cast<const Slice*>(column_block(cid).cell_ptr(row_idx));
+ column_string->insert_data(slice->data, strnlen(slice->data, slice->size));
+ } else {
+ column_string->insert_default();
+ }
+ }
+ break;
+ }
+ case OLAP_FIELD_TYPE_DATE: {
+ auto column_int = assert_cast<vectorized::ColumnVector<vectorized::Int64>*>(column);
+
+ for (uint16_t j = 0; j < _selected_size; ++j) {
+ if (!nullable_mark_array[j]) {
+ uint16_t row_idx = _selection_vector[j];
+ auto ptr = reinterpret_cast<const char*>(column_block(cid).cell_ptr(row_idx));
+
+ uint64_t value = 0;
+ value = *(unsigned char*)(ptr + 2);
+ value <<= 8;
+ value |= *(unsigned char*)(ptr + 1);
+ value <<= 8;
+ value |= *(unsigned char*)(ptr);
+ vectorized::VecDateTimeValue date;
+ date.from_olap_date(value);
+ (column_int)->insert_data(reinterpret_cast<char*>(&date), 0);
+ } else
+ column_int->insert_default();
+ }
+ break;
+ }
+ case OLAP_FIELD_TYPE_DATETIME: {
+ auto column_int = assert_cast<vectorized::ColumnVector<vectorized::Int64>*>(column);
+
+ for (uint16_t j = 0; j < _selected_size; ++j) {
+ if (!nullable_mark_array[j]) {
+ uint16_t row_idx = _selection_vector[j];
+ auto ptr = reinterpret_cast<const char*>(column_block(cid).cell_ptr(row_idx));
+
+ uint64_t value = *reinterpret_cast<const uint64_t*>(ptr);
+ vectorized::VecDateTimeValue data(value);
+ (column_int)->insert_data(reinterpret_cast<char*>(&data), 0);
+ } else {
+ column_int->insert_default();
+ }
+ }
+ break;
+ }
+ case OLAP_FIELD_TYPE_DECIMAL: {
+ auto column_decimal =
+ assert_cast<vectorized::ColumnDecimal<vectorized::Decimal128>*>(column);
+
+ for (uint16_t j = 0; j < _selected_size; ++j) {
+ if (!nullable_mark_array[j]) {
+ uint16_t row_idx = _selection_vector[j];
+ auto ptr = reinterpret_cast<const char*>(column_block(cid).cell_ptr(row_idx));
+
+ int64_t int_value = *(int64_t*)(ptr);
+ int32_t frac_value = *(int32_t*)(ptr + sizeof(int64_t));
+ DecimalV2Value data(int_value, frac_value);
+ column_decimal->insert_data(reinterpret_cast<char*>(&data), 0);
+ } else {
+ column_decimal->insert_default();
+ }
+ }
+ break;
+ }
+ case OLAP_FIELD_TYPE_INT: {
+ auto column_int = assert_cast<vectorized::ColumnVector<vectorized::Int32>*>(column);
+ insert_data_directly(cid, column_int);
+ break;
+ }
+ case OLAP_FIELD_TYPE_BOOL: {
+ auto column_int = assert_cast<vectorized::ColumnVector<vectorized::UInt8>*>(column);
+ insert_data_directly(cid, column_int);
+ break;
+ }
+ case OLAP_FIELD_TYPE_TINYINT: {
+ auto column_int = assert_cast<vectorized::ColumnVector<vectorized::Int8>*>(column);
+ insert_data_directly(cid, column_int);
+ break;
+ }
+ case OLAP_FIELD_TYPE_SMALLINT: {
+ auto column_int = assert_cast<vectorized::ColumnVector<vectorized::Int16>*>(column);
+ insert_data_directly(cid, column_int);
+ break;
+ }
+ case OLAP_FIELD_TYPE_BIGINT: {
+ auto column_int = assert_cast<vectorized::ColumnVector<vectorized::Int64>*>(column);
+ insert_data_directly(cid, column_int);
+ break;
+ }
+ case OLAP_FIELD_TYPE_LARGEINT: {
+ auto column_int = assert_cast<vectorized::ColumnVector<vectorized::Int128>*>(column);
+ insert_data_directly(cid, column_int);
+ break;
+ }
+ case OLAP_FIELD_TYPE_FLOAT: {
+ auto column_float = assert_cast<vectorized::ColumnVector<vectorized::Float32>*>(column);
+ insert_data_directly(cid, column_float);
+ break;
+ }
+ case OLAP_FIELD_TYPE_DOUBLE: {
+ auto column_float = assert_cast<vectorized::ColumnVector<vectorized::Float64>*>(column);
+ insert_data_directly(cid, column_float);
+ break;
+ }
+ default: {
+ DCHECK(false) << "Invalid type in RowBlockV2:" << _schema.column(cid)->type();
+ }
+ }
+}
+
+Status RowBlockV2::convert_to_vec_block(vectorized::Block* block) {
+ for (int i = 0; i < _schema.column_ids().size(); ++i) {
+ auto cid = _schema.column_ids()[i];
+ auto column = (*std::move(block->get_by_position(i).column)).assume_mutable();
+ _copy_data_to_column(cid, column);
+ }
+ _pool->clear();
+ return Status::OK();
+}
+
std::string RowBlockRow::debug_string() const {
std::stringstream ss;
ss << "[";
diff --git a/be/src/olap/row_block2.h b/be/src/olap/row_block2.h
index ece8d72..cdbf428 100644
--- a/be/src/olap/row_block2.h
+++ b/be/src/olap/row_block2.h
@@ -73,6 +73,9 @@ public:
// convert RowBlockV2 to RowBlock
Status convert_to_row_block(RowCursor* helper, RowBlock* dst);
+ // convert RowBlockV2 to vectorized::Block
+ Status convert_to_vec_block(vectorized::Block* block);
+
// low-level API to access memory for each column block(including data array and nullmap).
// `cid` must be one of `schema()->column_ids()`.
ColumnBlock column_block(ColumnId cid) const {
@@ -106,6 +109,8 @@ public:
std::string debug_string();
private:
+ void _copy_data_to_column(int cid, vectorized::MutableColumnPtr& mutable_column_ptr);
+
Schema _schema;
size_t _capacity;
// _column_vector_batches[cid] == null if cid is not in `_schema`.
diff --git a/be/src/olap/rowset/alpha_rowset_reader.h b/be/src/olap/rowset/alpha_rowset_reader.h
index 959a90b..fd90f37 100644
--- a/be/src/olap/rowset/alpha_rowset_reader.h
+++ b/be/src/olap/rowset/alpha_rowset_reader.h
@@ -65,6 +65,10 @@ public:
// It's ok, because we only get ref here, the block's owner is this reader.
OLAPStatus next_block(RowBlock** block) override;
+ OLAPStatus next_block(vectorized::Block *block) override {
+ return OLAP_ERR_DATA_EOF;
+ }
+
bool delete_flag() override;
Version version() override;
diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp
index ed264f3..a9fc448 100644
--- a/be/src/olap/rowset/beta_rowset_reader.cpp
+++ b/be/src/olap/rowset/beta_rowset_reader.cpp
@@ -27,6 +27,8 @@
#include "olap/rowset/segment_v2/segment_iterator.h"
#include "olap/schema.h"
+#include "vec/core/block.h"
+
namespace doris {
BetaRowsetReader::BetaRowsetReader(BetaRowsetSharedPtr rowset,
@@ -107,7 +109,8 @@ OLAPStatus BetaRowsetReader::init(RowsetReaderContext* read_context) {
}
seg_iterators.push_back(std::move(iter));
}
- std::list<RowwiseIterator*> iterators;
+
+ std::vector<RowwiseIterator*> iterators;
for (auto& owned_it : seg_iterators) {
// transfer ownership of segment iterator to `_iterator`
iterators.push_back(owned_it.release());
@@ -172,4 +175,41 @@ OLAPStatus BetaRowsetReader::next_block(RowBlock** block) {
return OLAP_SUCCESS;
}
+OLAPStatus BetaRowsetReader::next_block(vectorized::Block* block) {
+ SCOPED_RAW_TIMER(&_stats->block_fetch_ns);
+ bool is_first = true;
+
+ do {
+ // read next input block
+ {
+ _input_block->clear();
+ {
+ auto s = _iterator->next_batch(_input_block.get());
+ if (!s.ok()) {
+ if (s.is_end_of_file()) {
+ if (is_first) {
+ return OLAP_ERR_DATA_EOF;
+ } else {
+ break;
+ }
+ } else {
+ LOG(WARNING) << "failed to read next block: " << s.to_string();
+ return OLAP_ERR_ROWSET_READ_FAILED;
+ }
+ } else if (_input_block->selected_size() == 0) {
+ continue;
+ }
+ }
+ }
+
+ {
+ SCOPED_RAW_TIMER(&_stats->block_convert_ns);
+ _input_block->convert_to_vec_block(block);
+ }
+ is_first = false;
+ } while (block->rows() < _context->runtime_state->batch_size()); // here we should keep block.rows() < batch_size
+
+ return OLAP_SUCCESS;
+}
+
} // namespace doris
diff --git a/be/src/olap/rowset/beta_rowset_reader.h b/be/src/olap/rowset/beta_rowset_reader.h
index 34e7d9c..6074308 100644
--- a/be/src/olap/rowset/beta_rowset_reader.h
+++ b/be/src/olap/rowset/beta_rowset_reader.h
@@ -40,6 +40,7 @@ public:
// If parent_tracker is not null, the block we get from next_block() will have the parent_tracker.
// It's ok, because we only get ref here, the block's owner is this reader.
OLAPStatus next_block(RowBlock** block) override;
+ OLAPStatus next_block(vectorized::Block* block) override;
bool delete_flag() override { return _rowset->delete_flag(); }
diff --git a/be/src/olap/rowset/rowset_reader.h b/be/src/olap/rowset/rowset_reader.h
index b9e46d1..53af387 100644
--- a/be/src/olap/rowset/rowset_reader.h
+++ b/be/src/olap/rowset/rowset_reader.h
@@ -23,9 +23,14 @@
#include "olap/rowset/rowset.h"
#include "olap/rowset/rowset_reader_context.h"
+#include "vec/core/block.h"
namespace doris {
+namespace vectorized {
+class Block;
+}
+
class RowBlock;
class RowsetReader;
using RowsetReaderSharedPtr = std::shared_ptr<RowsetReader>;
@@ -33,6 +38,7 @@ using RowsetReaderSharedPtr = std::shared_ptr<RowsetReader>;
class RowsetReader {
public:
enum RowsetReaderType { ALPHA, BETA };
+
virtual ~RowsetReader() {}
// reader init
@@ -45,6 +51,8 @@ public:
// Others when error happens.
virtual OLAPStatus next_block(RowBlock** block) = 0;
+ virtual OLAPStatus next_block(vectorized::Block* block) = 0;
+
virtual bool delete_flag() = 0;
virtual Version version() = 0;
diff --git a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp
index 6fae7eb..7825dcc 100644
--- a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp
+++ b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp
@@ -19,9 +19,12 @@
#include "common/logging.h"
#include "gutil/strings/substitute.h" // for Substitute
-#include "olap/rowset/segment_v2/bitshuffle_page.h"
#include "runtime/mem_pool.h"
#include "util/slice.h" // for Slice
+#include "vec/columns/column_vector.h"
+#include "vec/columns/column_string.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/predicate_column.h"
namespace doris {
namespace segment_v2 {
@@ -220,6 +223,8 @@ Status BinaryDictPageDecoder::init() {
return Status::OK();
}
+BinaryDictPageDecoder::~BinaryDictPageDecoder() {}
+
Status BinaryDictPageDecoder::seek_to_position_in_page(size_t pos) {
return _data_page_decoder->seek_to_position_in_page(pos);
}
@@ -230,8 +235,53 @@ bool BinaryDictPageDecoder::is_dict_encoding() const {
void BinaryDictPageDecoder::set_dict_decoder(PageDecoder* dict_decoder) {
_dict_decoder = (BinaryPlainPageDecoder*)dict_decoder;
+ _bit_shuffle_ptr = reinterpret_cast<BitShufflePageDecoder<OLAP_FIELD_TYPE_INT>*>(_data_page_decoder.get());
};
+Status BinaryDictPageDecoder::next_batch(size_t* n, vectorized::MutableColumnPtr &dst) {
+ if (_encoding_type == PLAIN_ENCODING) {
+ return _data_page_decoder->next_batch(n, dst);
+ }
+ // dictionary encoding
+ DCHECK(_parsed);
+ DCHECK(_dict_decoder != nullptr) << "dict decoder pointer is nullptr";
+
+ if (PREDICT_FALSE(*n == 0 || _bit_shuffle_ptr->_cur_index >= _bit_shuffle_ptr->_num_elements)) {
+ *n = 0;
+ return Status::OK();
+ }
+
+ size_t max_fetch = std::min(*n, static_cast<size_t>(_bit_shuffle_ptr->_num_elements - _bit_shuffle_ptr->_cur_index));
+ *n = max_fetch;
+
+ const int32_t* data_array = reinterpret_cast<const int32_t*>(_bit_shuffle_ptr->_chunk.data);
+ size_t start_index = _bit_shuffle_ptr->_cur_index;
+
+ // todo(wb) support nullable
+ if (dst->is_predicate_column()) {
+ // cast columnptr to columnstringvalue just for avoid virtual function call overhead
+ vectorized::ColumnStringValue& string_value_vector = reinterpret_cast<vectorized::ColumnStringValue&>(*dst);
+ for (int i = 0; i < max_fetch; i++, start_index++) {
+ int32_t codeword = data_array[start_index];
+ uint32_t start_offset = _start_offset_array[codeword];
+ uint32_t str_len = _len_array[codeword];
+ string_value_vector.insert_data(&_dict_decoder->_data[start_offset], str_len);
+ }
+ } else {
+ // todo(wb) research whether using batch memcpy to insert columnString can has better performance when data set is big
+ for (int i = 0; i < max_fetch; i++, start_index++) {
+ int32_t codeword = data_array[start_index];
+ const uint32_t start_offset = _start_offset_array[codeword];
+ const uint32_t str_len = _len_array[codeword];
+ dst->insert_data(&_dict_decoder->_data[start_offset], str_len);
+ }
+ }
+ _bit_shuffle_ptr->_cur_index += max_fetch;
+
+ return Status::OK();
+
+}
+
Status BinaryDictPageDecoder::next_batch(size_t* n, ColumnBlockView* dst) {
if (_encoding_type == PLAIN_ENCODING) {
return _data_page_decoder->next_batch(n, dst);
diff --git a/be/src/olap/rowset/segment_v2/binary_dict_page.h b/be/src/olap/rowset/segment_v2/binary_dict_page.h
index 9290466..25a70b9 100644
--- a/be/src/olap/rowset/segment_v2/binary_dict_page.h
+++ b/be/src/olap/rowset/segment_v2/binary_dict_page.h
@@ -33,6 +33,7 @@
#include "olap/types.h"
#include "runtime/mem_pool.h"
#include "runtime/mem_tracker.h"
+#include "olap/rowset/segment_v2/bitshuffle_page.h"
namespace doris {
namespace segment_v2 {
@@ -106,6 +107,8 @@ public:
Status next_batch(size_t* n, ColumnBlockView* dst) override;
+ Status next_batch(size_t* n, vectorized::MutableColumnPtr &dst) override;
+
size_t count() const override { return _data_page_decoder->count(); }
size_t current_index() const override { return _data_page_decoder->current_index(); }
@@ -114,15 +117,21 @@ public:
void set_dict_decoder(PageDecoder* dict_decoder);
+ ~BinaryDictPageDecoder();
+
private:
Slice _data;
PageDecoderOptions _options;
std::unique_ptr<PageDecoder> _data_page_decoder;
const BinaryPlainPageDecoder* _dict_decoder = nullptr;
+ BitShufflePageDecoder<OLAP_FIELD_TYPE_INT>* _bit_shuffle_ptr = nullptr;
bool _parsed;
EncodingTypePB _encoding_type;
// use as data buf.
std::unique_ptr<ColumnVectorBatch> _batch;
+
+ uint32_t* _start_offset_array = nullptr;
+ uint32_t* _len_array = nullptr;
};
} // namespace segment_v2
diff --git a/be/src/olap/rowset/segment_v2/binary_plain_page.h b/be/src/olap/rowset/segment_v2/binary_plain_page.h
index 0747df2..25ce9e0 100644
--- a/be/src/olap/rowset/segment_v2/binary_plain_page.h
+++ b/be/src/olap/rowset/segment_v2/binary_plain_page.h
@@ -227,6 +227,35 @@ public:
return Status::OK();
}
+ Status next_batch(size_t* n, vectorized::MutableColumnPtr &dst) override {
+ DCHECK(_parsed);
+ if (PREDICT_FALSE(*n == 0 || _cur_idx >= _num_elems)) {
+ *n = 0;
+ return Status::OK();
+ }
+ const size_t max_fetch = std::min(*n, static_cast<size_t>(_num_elems - _cur_idx));
+
+ if (dst->is_predicate_column()) {
+ // todo(wb) padding sv here for better comparison performance
+ for (size_t i = 0; i < max_fetch; i++, _cur_idx++) {
+ const uint32_t start_offset = offset(_cur_idx);
+ uint32_t len = offset(_cur_idx + 1) - start_offset;
+ StringValue sv(const_cast<char*>(&_data[start_offset]), len);
+ dst->insert_data(reinterpret_cast<char*>(&sv), 0);
+ }
+ } else {
+ for (size_t i = 0; i < max_fetch; i++, _cur_idx++) {
+ // todo(wb) need more test case and then improve here
+ const uint32_t start_offset = offset(_cur_idx);
+ uint32_t len = offset(_cur_idx + 1) - start_offset;
+ dst->insert_data(&_data[start_offset], len);
+ }
+ }
+
+ *n = max_fetch;
+ return Status::OK();
+ };
+
size_t count() const override {
DCHECK(_parsed);
return _num_elems;
@@ -263,6 +292,8 @@ private:
// Index of the currently seeked element in the page.
uint32_t _cur_idx;
+ friend class BinaryDictPageDecoder;
+ friend class FileColumnIterator;
};
} // namespace segment_v2
diff --git a/be/src/olap/rowset/segment_v2/binary_prefix_page.h b/be/src/olap/rowset/segment_v2/binary_prefix_page.h
index 7e51b82..7d7bcc9 100644
--- a/be/src/olap/rowset/segment_v2/binary_prefix_page.h
+++ b/be/src/olap/rowset/segment_v2/binary_prefix_page.h
@@ -113,6 +113,10 @@ public:
Status next_batch(size_t* n, ColumnBlockView* dst) override;
+ Status next_batch(size_t* n, vectorized::MutableColumnPtr &dst) override {
+ return Status::NotSupported("binary prefix page not implement vec op now");
+ };
+
size_t count() const override {
DCHECK(_parsed);
return _num_values;
diff --git a/be/src/olap/rowset/segment_v2/bitshuffle_page.h b/be/src/olap/rowset/segment_v2/bitshuffle_page.h
index cd54b6b..5afbb7d 100644
--- a/be/src/olap/rowset/segment_v2/bitshuffle_page.h
+++ b/be/src/olap/rowset/segment_v2/bitshuffle_page.h
@@ -348,6 +348,53 @@ public:
return Status::OK();
}
+ Status next_batch(size_t* n, vectorized::MutableColumnPtr &dst) override {
+ DCHECK(_parsed);
+ if (PREDICT_FALSE(*n == 0 || _cur_index >= _num_elements)) {
+ *n = 0;
+ return Status::OK();
+ }
+
+ size_t max_fetch = std::min(*n, static_cast<size_t>(_num_elements - _cur_index));
+
+ int begin = _cur_index;
+ int end = _cur_index + max_fetch;
+
+ // todo(wb) Try to eliminate type judgment in pagedecoder
+ if (dst->is_column_decimal()) { // decimal non-predicate column
+ for (; begin < end; begin++) {
+ const char* cur_ptr = (const char*)&_chunk.data[begin * SIZE_OF_TYPE];
+ int64_t int_value = *(int64_t*)(cur_ptr);
+ int32_t frac_value = *(int32_t*)(cur_ptr + sizeof(int64_t));
+ DecimalV2Value data(int_value, frac_value);
+ dst->insert_data(reinterpret_cast<char*>(&data), 0);
+ }
+ } else if (dst->is_date_type()) {
+ for (; begin < end; begin++) {
+ const char* cur_ptr = (const char*)&_chunk.data[begin * SIZE_OF_TYPE];
+ uint64_t value = 0;
+ value = *(unsigned char*)(cur_ptr + 2);
+ value <<= 8;
+ value |= *(unsigned char*)(cur_ptr + 1);
+ value <<= 8;
+ value |= *(unsigned char*)(cur_ptr);
+ DateTimeValue date;
+ date.from_olap_date(value);
+ dst->insert_data(reinterpret_cast<char*>(&date), 0);
+ }
+ } else {
+ // todo(wb) need performance test here, may be batch memory copy?
+ for (; begin < end; begin++) {
+ dst->insert_data((const char*)&_chunk.data[begin * SIZE_OF_TYPE], 0);
+ }
+ }
+
+ *n = max_fetch;
+ _cur_index += max_fetch;
+
+ return Status::OK();
+ };
+
Status peek_next_batch(size_t* n, ColumnBlockView* dst) override {
return next_batch<false>(n, dst);
}
@@ -393,6 +440,7 @@ private:
int _size_of_element;
size_t _cur_index;
Chunk _chunk;
+ friend class BinaryDictPageDecoder;
};
} // namespace segment_v2
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 3bbbeab..c08b328 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -30,6 +30,8 @@
#include "util/block_compression.h"
#include "util/coding.h" // for get_varint32
#include "util/rle_encoding.h" // for RleDecoder
+#include "vec/core/types.h"
+#include "vec/runtime/vdatetime_value.h" //for VecDateTime
namespace doris {
namespace segment_v2 {
@@ -578,6 +580,57 @@ Status FileColumnIterator::next_batch(size_t* n, ColumnBlockView* dst, bool* has
return Status::OK();
}
+Status FileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr &dst, bool* has_null) {
+ size_t curr_size = dst->byte_size();
+ size_t remaining = *n;
+ *has_null = false;
+ while (remaining > 0) {
+ if (!_page->has_remaining()) {
+ bool eos = false;
+ RETURN_IF_ERROR(_load_next_page(&eos));
+ if (eos) {
+ break;
+ }
+ }
+
+ // number of rows to be read from this page
+ size_t nrows_in_page = std::min(remaining, _page->remaining());
+ size_t nrows_to_read = nrows_in_page;
+ if (_page->has_null) {
+ while (nrows_to_read > 0) {
+ bool is_null = false;
+ size_t this_run = _page->null_decoder.GetNextRun(&is_null, nrows_to_read);
+ // we use num_rows only for CHECK
+ size_t num_rows = this_run;
+ if (!is_null) {
+ RETURN_IF_ERROR(_page->data_decoder->next_batch(&num_rows, dst));
+ DCHECK_EQ(this_run, num_rows);
+ } else {
+ *has_null = true;
+ // todo(wb) add a DCHECK here to check whether type is column nullable
+ for (size_t x = 0; x < this_run; x++) {
+ dst->insert_data(nullptr, 0); // todo(wb) vectorized here
+ }
+ }
+
+ nrows_to_read -= this_run;
+ _page->offset_in_page += this_run;
+ _current_ordinal += this_run;
+ }
+ } else {
+ RETURN_IF_ERROR(_page->data_decoder->next_batch(&nrows_to_read, dst));
+ DCHECK_EQ(nrows_to_read, nrows_in_page);
+
+ _page->offset_in_page += nrows_to_read;
+ _current_ordinal += nrows_to_read;
+ }
+ remaining -= nrows_in_page;
+ }
+ *n -= remaining;
+ _opts.stats->bytes_read += (dst->byte_size() - curr_size) + BitmapSize(*n);
+ return Status::OK();
+}
+
Status FileColumnIterator::_load_next_page(bool* eos) {
_page_iter.next();
if (!_page_iter.valid()) {
@@ -715,5 +768,55 @@ Status DefaultValueColumnIterator::next_batch(size_t* n, ColumnBlockView* dst, b
return Status::OK();
}
+void DefaultValueColumnIterator::insert_default_data(vectorized::MutableColumnPtr &dst, size_t n) {
+ vectorized::Int128 int128;
+ char* data_ptr = (char*)&int128;
+ size_t data_len = sizeof(int128);
+
+ auto type = _type_info->type();
+ if (type == OLAP_FIELD_TYPE_DATE) {
+ assert(_type_size == sizeof(FieldTypeTraits<OLAP_FIELD_TYPE_DATE>::CppType)); //uint24_t
+ std::string str = FieldTypeTraits<OLAP_FIELD_TYPE_DATE>::to_string(_mem_value);
+
+ vectorized::VecDateTimeValue value;
+ value.from_date_str(str.c_str(), str.length());
+ value.cast_to_date();
+ //TODO: here is int128 = int64
+ int128 = binary_cast<vectorized::VecDateTimeValue, vectorized::Int64>(value);
+ } else if (type == OLAP_FIELD_TYPE_DATETIME) {
+ assert(_type_size == sizeof(FieldTypeTraits<OLAP_FIELD_TYPE_DATETIME>::CppType)); //int64_t
+ std::string str = FieldTypeTraits<OLAP_FIELD_TYPE_DATETIME>::to_string(_mem_value);
+
+ vectorized::VecDateTimeValue value;
+ value.from_date_str(str.c_str(), str.length());
+ value.to_datetime();
+
+ int128 = binary_cast<vectorized::VecDateTimeValue, vectorized::Int64>(value);
+ } else if (type == OLAP_FIELD_TYPE_DECIMAL) {
+ assert(_type_size == sizeof(FieldTypeTraits<OLAP_FIELD_TYPE_DECIMAL>::CppType)); //decimal12_t
+ decimal12_t* d = (decimal12_t*)_mem_value;
+ int128 = DecimalV2Value(d->integer, d->fraction).value();
+ } else {
+ data_ptr = (char*)_mem_value;
+ data_len = _type_size;
+ }
+
+ for (size_t i = 0; i < n; ++i) {
+ dst->insert_data(data_ptr, data_len);
+ }
+}
+
+Status DefaultValueColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr &dst, bool* has_null) {
+ if (_is_default_value_null) {
+ *has_null = true;
+ dst->insert_many_defaults(*n);
+ } else {
+ *has_null = false;
+ insert_default_data(dst, *n);
+ }
+
+ return Status::OK();
+}
+
} // namespace segment_v2
} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h
index 0b3588c..98afe44 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -215,11 +215,20 @@ public:
return next_batch(n, dst, &has_null);
}
+ Status next_batch(size_t* n, vectorized::MutableColumnPtr &dst) {
+ bool has_null;
+ return next_batch(n, dst, &has_null);
+ }
+
// After one seek, we can call this function many times to read data
// into ColumnBlockView. when read string type data, memory will allocated
// from MemPool
virtual Status next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) = 0;
+ virtual Status next_batch(size_t* n, vectorized::MutableColumnPtr &dst, bool* has_null) {
+ return Status::NotSupported("not implement");
+ }
+
virtual ordinal_t get_current_ordinal() const = 0;
virtual Status get_row_ranges_by_zone_map(CondColumn* cond_column, CondColumn* delete_condition,
@@ -268,6 +277,8 @@ public:
Status next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) override;
+ Status next_batch(size_t* n, vectorized::MutableColumnPtr &dst, bool* has_null) override;
+
ordinal_t get_current_ordinal() const override { return _current_ordinal; }
// get row ranges by zone map
@@ -401,11 +412,20 @@ public:
return Status::OK();
}
+ Status next_batch(size_t* n, vectorized::MutableColumnPtr &dst) {
+ bool has_null;
+ return next_batch(n, dst, &has_null);
+ }
+
Status next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) override;
+ Status next_batch(size_t* n, vectorized::MutableColumnPtr &dst, bool* has_null) override;
+
ordinal_t get_current_ordinal() const override { return _current_rowid; }
private:
+ void insert_default_data(vectorized::MutableColumnPtr &dst, size_t n);
+
bool _has_default_value;
std::string _default_value;
bool _is_nullable;
diff --git a/be/src/olap/rowset/segment_v2/empty_segment_iterator.cpp b/be/src/olap/rowset/segment_v2/empty_segment_iterator.cpp
index 61c0fab..f367c2d 100644
--- a/be/src/olap/rowset/segment_v2/empty_segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/empty_segment_iterator.cpp
@@ -30,5 +30,9 @@ Status EmptySegmentIterator::next_batch(RowBlockV2* block) {
return Status::EndOfFile("no more data in segment");
}
+Status EmptySegmentIterator::next_batch(vectorized::Block* block) {
+ return Status::EndOfFile("no more data in segment");
+}
+
} // namespace segment_v2
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/empty_segment_iterator.h b/be/src/olap/rowset/segment_v2/empty_segment_iterator.h
index cb6c48c..3e1a4f9 100644
--- a/be/src/olap/rowset/segment_v2/empty_segment_iterator.h
+++ b/be/src/olap/rowset/segment_v2/empty_segment_iterator.h
@@ -32,10 +32,11 @@ public:
Status init(const StorageReadOptions& opts) override { return Status::OK(); }
const Schema& schema() const override { return _schema; }
Status next_batch(RowBlockV2* row_block) override;
+ Status next_batch(vectorized::Block* block) override;
private:
Schema _schema;
};
} // namespace segment_v2
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/frame_of_reference_page.h b/be/src/olap/rowset/segment_v2/frame_of_reference_page.h
index 5d0c0b1..df8fe06 100644
--- a/be/src/olap/rowset/segment_v2/frame_of_reference_page.h
+++ b/be/src/olap/rowset/segment_v2/frame_of_reference_page.h
@@ -161,6 +161,10 @@ public:
return Status::OK();
}
+ Status next_batch(size_t* n, vectorized::MutableColumnPtr &dst) override {
+ return Status::NotSupported("frame page not implement vec op now");
+ };
+
Status peek_next_batch(size_t* n, ColumnBlockView* dst) override {
return next_batch<false>(n, dst);
}
diff --git a/be/src/olap/rowset/segment_v2/page_decoder.h b/be/src/olap/rowset/segment_v2/page_decoder.h
index feed37f..ab5b482 100644
--- a/be/src/olap/rowset/segment_v2/page_decoder.h
+++ b/be/src/olap/rowset/segment_v2/page_decoder.h
@@ -19,6 +19,7 @@
#include "common/status.h" // for Status
#include "olap/column_block.h" // for ColumnBlockView
+#include "vec/columns/column.h"
namespace doris {
namespace segment_v2 {
@@ -81,6 +82,10 @@ public:
// allocated in the column_vector_view's mem_pool.
virtual Status next_batch(size_t* n, ColumnBlockView* dst) = 0;
+ virtual Status next_batch(size_t* n, vectorized::MutableColumnPtr &dst) {
+ return Status::NotSupported("not implement vec op now");
+ }
+
// Same as `next_batch` except for not moving forward the cursor.
// When read array's ordinals in `ArrayFileColumnIterator`, we want to read one extra ordinal
// but do not want to move forward the cursor.
diff --git a/be/src/olap/rowset/segment_v2/plain_page.h b/be/src/olap/rowset/segment_v2/plain_page.h
index 555c9af..09e4280 100644
--- a/be/src/olap/rowset/segment_v2/plain_page.h
+++ b/be/src/olap/rowset/segment_v2/plain_page.h
@@ -186,6 +186,10 @@ public:
Status next_batch(size_t* n, ColumnBlockView* dst) override { return next_batch<true>(n, dst); }
+ Status next_batch(size_t* n, vectorized::MutableColumnPtr &dst) override {
+ return Status::NotSupported("plain page not implement vec op now");
+ };
+
template <bool forward_index>
inline Status next_batch(size_t* n, ColumnBlockView* dst) {
DCHECK(_parsed);
diff --git a/be/src/olap/rowset/segment_v2/rle_page.h b/be/src/olap/rowset/segment_v2/rle_page.h
index ed20aaf..52a9613 100644
--- a/be/src/olap/rowset/segment_v2/rle_page.h
+++ b/be/src/olap/rowset/segment_v2/rle_page.h
@@ -230,6 +230,29 @@ public:
return Status::OK();
}
+ Status next_batch(size_t* n, vectorized::MutableColumnPtr &dst) override {
+ DCHECK(_parsed);
+ if (PREDICT_FALSE(*n == 0 || _cur_index >= _num_elements)) {
+ *n = 0;
+ return Status::OK();
+ }
+
+ size_t to_fetch = std::min(*n, static_cast<size_t>(_num_elements - _cur_index));
+ size_t remaining = to_fetch;
+ bool result = false;
+ CppType value;
+ while (remaining > 0) {
+ result = _rle_decoder.Get(&value);
+ DCHECK(result);
+ dst->insert_data((char*)(&value), SIZE_OF_TYPE);
+ remaining--;
+ }
+
+ _cur_index += to_fetch;
+ *n = to_fetch;
+ return Status::OK();
+ };
+
size_t count() const override { return _num_elements; }
size_t current_index() const override { return _cur_index; }
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index cb7e0d5..d19f56e 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -581,5 +581,10 @@ Status SegmentIterator::next_batch(RowBlockV2* block) {
return Status::OK();
}
+Status SegmentIterator::next_batch(vectorized::Block* block) {
+ //TODO
+ return Status::NotSupported("not implement now");
+}
+
} // namespace segment_v2
} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.h b/be/src/olap/rowset/segment_v2/segment_iterator.h
index 0da6ae4..fabefd5 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.h
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.h
@@ -50,8 +50,11 @@ public:
SegmentIterator(std::shared_ptr<Segment> segment, const Schema& _schema,
std::shared_ptr<MemTracker> parent);
~SegmentIterator() override;
+
Status init(const StorageReadOptions& opts) override;
Status next_batch(RowBlockV2* row_block) override;
+ Status next_batch(vectorized::Block* block) override;
+
const Schema& schema() const override { return _schema; }
bool is_lazy_materialization_read() const override { return _lazy_materialization_read; }
uint64_t data_id() const { return _segment->id(); }
diff --git a/be/src/olap/schema.cpp b/be/src/olap/schema.cpp
index ab00e37..d588b58 100644
--- a/be/src/olap/schema.cpp
+++ b/be/src/olap/schema.cpp
@@ -103,4 +103,48 @@ Schema::~Schema() {
}
}
+vectorized::DataTypePtr Schema::get_data_type_ptr(FieldType type) {
+ switch (type) {
+ case OLAP_FIELD_TYPE_TINYINT:
+ return std::make_shared<vectorized::DataTypeInt8>();
+
+ case OLAP_FIELD_TYPE_SMALLINT:
+ return std::make_shared<vectorized::DataTypeInt16>();
+
+ case OLAP_FIELD_TYPE_INT:
+ return std::make_shared<vectorized::DataTypeInt32>();
+
+ case OLAP_FIELD_TYPE_FLOAT:
+ return std::make_shared<vectorized::DataTypeFloat32>();
+
+ case OLAP_FIELD_TYPE_BIGINT:
+ return std::make_shared<vectorized::DataTypeInt64>();
+
+ case OLAP_FIELD_TYPE_LARGEINT:
+ return std::make_shared<vectorized::DataTypeInt128>();
+
+ case OLAP_FIELD_TYPE_DATE:
+ return std::make_shared<vectorized::DataTypeDate>();
+
+ case OLAP_FIELD_TYPE_DATETIME:
+ return std::make_shared<vectorized::DataTypeDateTime>();
+
+ case OLAP_FIELD_TYPE_DOUBLE:
+ return std::make_shared<vectorized::DataTypeFloat64>();
+
+ case OLAP_FIELD_TYPE_CHAR:
+ case OLAP_FIELD_TYPE_VARCHAR:
+ case OLAP_FIELD_TYPE_HLL:
+ return std::make_shared<vectorized::DataTypeString>();
+
+ case OLAP_FIELD_TYPE_DECIMAL:
+ return std::make_shared<vectorized::DataTypeDecimal<vectorized::Decimal128>>(27, 9);
+
+ default:
+ DCHECK(false);
+ }
+ // For llvm complain
+ return nullptr;
+}
+
} // namespace doris
diff --git a/be/src/olap/schema.h b/be/src/olap/schema.h
index 39e3c79..3fd90bf 100644
--- a/be/src/olap/schema.h
+++ b/be/src/olap/schema.h
@@ -100,6 +100,10 @@ public:
~Schema();
+ static vectorized::DataTypePtr get_data_type_ptr(FieldType type);
+
+ const std::vector<Field*>& columns() const { return _cols; }
+
const Field* column(ColumnId cid) const { return _cols[cid]; }
Field* mutable_column(ColumnId cid) const { return _cols[cid]; }
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 072a415..5b407d5 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -18,6 +18,8 @@
#include "olap/tablet_schema.h"
#include "tablet_meta.h"
+#include "vec/core/block.h"
+#include "vec/data_types/data_type.h"
namespace doris {
@@ -490,6 +492,16 @@ void TabletSchema::init_field_index_for_test() {
}
}
+vectorized::Block TabletSchema::create_block(const std::vector<uint32_t>& return_columns) const {
+ vectorized::Block block;
+ for (int i = 0; i < return_columns.size(); ++i) {
+ const auto& col = _cols[return_columns[i]];
+ auto data_type = vectorized::IDataType::from_olap_engine(col.type(), col.is_nullable());
+ block.insert({data_type->create_column(), data_type, col.name()});
+ }
+ return block;
+}
+
bool operator==(const TabletColumn& a, const TabletColumn& b) {
if (a._unique_id != b._unique_id) return false;
if (a._col_name != b._col_name) return false;
@@ -548,4 +560,5 @@ bool operator!=(const TabletSchema& a, const TabletSchema& b) {
return !(a == b);
}
+
} // namespace doris
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index 0cb894f..ffd8c34 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -26,6 +26,9 @@
#include "olap/types.h"
namespace doris {
+namespace vectorized {
+class Block;
+}
class TabletColumn {
public:
@@ -141,6 +144,7 @@ public:
inline void set_delete_sign_idx(int32_t delete_sign_idx) { _delete_sign_idx = delete_sign_idx; }
inline bool has_sequence_col() const { return _sequence_col_idx != -1; }
inline int32_t sequence_col_idx() const { return _sequence_col_idx; }
+ vectorized::Block create_block(const std::vector<uint32_t>& return_columns) const;
private:
// Only for unit test
diff --git a/be/src/olap/tuple_reader.cpp b/be/src/olap/tuple_reader.cpp
index fa44e4a..f1113d1 100644
--- a/be/src/olap/tuple_reader.cpp
+++ b/be/src/olap/tuple_reader.cpp
@@ -198,7 +198,7 @@ OLAPStatus TupleReader::_unique_key_next_row(RowCursor* row_cursor, MemPool* mem
cur_delete_flag = _next_delete_flag;
}
- // if reader needs to filter delete row and current delete_flag is ture,
+ // if reader needs to filter delete row and current delete_flag is true,
// then continue
if (!(cur_delete_flag && _filter_delete)) {
break;
diff --git a/be/src/runtime/datetime_value.cpp b/be/src/runtime/datetime_value.cpp
index 9ec8ccb..b545f55 100644
--- a/be/src/runtime/datetime_value.cpp
+++ b/be/src/runtime/datetime_value.cpp
@@ -1466,6 +1466,8 @@ bool DateTimeValue::from_date_format_str(const char* format, int format_len, con
}
bool DateTimeValue::date_add_interval(const TimeInterval& interval, TimeUnit unit) {
+ if (!is_valid_date()) return false;
+
int sign = interval.is_neg ? -1 : 1;
switch (unit) {
case MICROSECOND:
diff --git a/be/src/runtime/datetime_value.h b/be/src/runtime/datetime_value.h
index 096a24d..4f3a310 100644
--- a/be/src/runtime/datetime_value.h
+++ b/be/src/runtime/datetime_value.h
@@ -30,7 +30,7 @@
#include "udf/udf.h"
#include "util/hash_util.hpp"
#include "util/timezone_utils.h"
-
+#include "vec/runtime/vdatetime_value.h"
namespace doris {
enum TimeUnit {
@@ -566,7 +566,9 @@ public:
private:
// Used to make sure sizeof DateTimeValue
friend class UnusedClass;
-
+ friend void doris::vectorized::VecDateTimeValue::convert_vec_dt_to_dt(DateTimeValue* dt);
+ friend void doris::vectorized::VecDateTimeValue::convert_dt_to_vec_dt(DateTimeValue* dt);
+
void from_packed_time(int64_t packed_time) {
_microsecond = packed_time % (1LL << 24);
int64_t ymdhms = packed_time >> 24;
diff --git a/be/src/runtime/descriptors.cpp b/be/src/runtime/descriptors.cpp
index ab4fa97..95d0a79 100644
--- a/be/src/runtime/descriptors.cpp
+++ b/be/src/runtime/descriptors.cpp
@@ -25,6 +25,10 @@
#include "gen_cpp/Descriptors_types.h"
#include "gen_cpp/descriptors.pb.h"
+#include "vec/core/columns_with_type_and_name.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/data_types/data_type_nullable.h"
+
namespace doris {
using boost::algorithm::join;
@@ -80,6 +84,21 @@ void SlotDescriptor::to_protobuf(PSlotDescriptor* pslot) const {
pslot->set_is_materialized(_is_materialized);
}
+vectorized::MutableColumnPtr SlotDescriptor::get_empty_mutable_column() const {
+ auto data_column = type().get_data_type_ptr()->create_column();
+ if (is_nullable()) {
+ return doris::vectorized::ColumnNullable::create(std::move(data_column), doris::vectorized::ColumnUInt8::create());
+ }
+ return data_column;
+}
+
+vectorized::DataTypePtr SlotDescriptor::get_data_type_ptr() const {
+ if (is_nullable()) {
+ return std::make_shared<vectorized::DataTypeNullable>(type().get_data_type_ptr());
+ }
+ return type().get_data_type_ptr();
+}
+
std::string SlotDescriptor::debug_string() const {
std::stringstream out;
out << "Slot(id=" << _id << " type=" << _type << " col=" << _col_pos
@@ -378,7 +397,7 @@ void RowDescriptor::to_thrift(std::vector<TTupleId>* row_tuple_ids) {
}
void RowDescriptor::to_protobuf(
- google::protobuf::RepeatedField<google::protobuf::int32>* row_tuple_ids) {
+ google::protobuf::RepeatedField<google::protobuf::int32>* row_tuple_ids) const {
row_tuple_ids->Clear();
for (auto desc : _tuple_desc_map) {
row_tuple_ids->Add(desc->id());
diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h
index d668d42..97f9712 100644
--- a/be/src/runtime/descriptors.h
+++ b/be/src/runtime/descriptors.h
@@ -103,6 +103,9 @@ public:
std::string debug_string() const;
+ doris::vectorized::MutableColumnPtr get_empty_mutable_column() const;
+
+ doris::vectorized::DataTypePtr get_data_type_ptr() const;
private:
friend class DescriptorTbl;
friend class TupleDescriptor;
@@ -405,7 +408,7 @@ public:
// Populate row_tuple_ids with our ids.
void to_thrift(std::vector<TTupleId>* row_tuple_ids);
- void to_protobuf(google::protobuf::RepeatedField<google::protobuf::int32>* row_tuple_ids);
+ void to_protobuf(google::protobuf::RepeatedField<google::protobuf::int32>* row_tuple_ids) const;
// Return true if the tuple ids of this descriptor are a prefix
// of the tuple ids of other_desc.
diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h
index 3cc2def..0b51e47 100644
--- a/be/src/runtime/exec_env.h
+++ b/be/src/runtime/exec_env.h
@@ -93,6 +93,7 @@ public:
const std::string& token() const;
ExternalScanContextMgr* external_scan_context_mgr() { return _external_scan_context_mgr; }
DataStreamMgr* stream_mgr() { return _stream_mgr; }
+ doris::vectorized::VDataStreamMgr* vstream_mgr() { return _vstream_mgr; }
ResultBufferMgr* result_mgr() { return _result_mgr; }
ResultQueueMgr* result_queue_mgr() { return _result_queue_mgr; }
ClientCache<BackendServiceClient>* client_cache() { return _backend_client_cache; }
@@ -163,6 +164,7 @@ private:
// Leave protected so that subclasses can override
ExternalScanContextMgr* _external_scan_context_mgr = nullptr;
DataStreamMgr* _stream_mgr = nullptr;
+ doris::vectorized::VDataStreamMgr* _vstream_mgr = nullptr;
ResultBufferMgr* _result_mgr = nullptr;
ResultQueueMgr* _result_queue_mgr = nullptr;
ClientCache<BackendServiceClient>* _backend_client_cache = nullptr;
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index a172bb6..35630d0 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -63,6 +63,7 @@
#include "util/parse_util.h"
#include "util/pretty_printer.h"
#include "util/priority_thread_pool.hpp"
+#include "vec/runtime/vdata_stream_mgr.h"
namespace doris {
@@ -85,6 +86,7 @@ Status ExecEnv::_init(const std::vector<StorePath>& store_paths) {
_store_paths = store_paths;
_external_scan_context_mgr = new ExternalScanContextMgr(this);
_stream_mgr = new DataStreamMgr();
+ _vstream_mgr = new doris::vectorized::VDataStreamMgr();
_result_mgr = new ResultBufferMgr();
_result_queue_mgr = new ResultQueueMgr();
_backend_client_cache = new BackendServiceClientCache(config::max_client_cache_size_per_host);
diff --git a/be/src/runtime/fold_constant_executor.cpp b/be/src/runtime/fold_constant_executor.cpp
index 94d3d9e..cd6d5ff 100644
--- a/be/src/runtime/fold_constant_executor.cpp
+++ b/be/src/runtime/fold_constant_executor.cpp
@@ -28,6 +28,9 @@
#include "common/object_pool.h"
#include "common/status.h"
+#include "vec/exprs/vexpr.h"
+#include "vec/exprs/vexpr_context.h"
+
#include "gen_cpp/internal_service.pb.h"
#include "gen_cpp/PaloInternalService_types.h"
@@ -96,6 +99,66 @@ Status FoldConstantExecutor::fold_constant_expr(
return Status::OK();
}
+Status FoldConstantExecutor::fold_constant_vexpr(
+ const TFoldConstantParams& params, PConstantExprResult* response) {
+ const auto& expr_map = params.expr_map;
+ auto expr_result_map = response->mutable_expr_result_map();
+
+ TQueryGlobals query_globals = params.query_globals;
+ // init
+ Status status = _init(query_globals);
+ if (UNLIKELY(!status.ok())) {
+ LOG(WARNING) << "Failed to init mem trackers, msg: " << status.get_error_msg();
+ return status;
+ }
+
+ for (const auto& m : expr_map) {
+ PExprResultMap pexpr_result_map;
+ for (const auto& n : m.second) {
+ vectorized::VExprContext* ctx = nullptr;
+ const TExpr& texpr = n.second;
+ // create expr tree from TExpr
+ RETURN_IF_ERROR(vectorized::VExpr::create_expr_tree(&_pool, texpr, &ctx));
+ // prepare and open context
+ status = _prepare_and_open(ctx);
+ if (UNLIKELY(!status.ok())) {
+ LOG(WARNING) << "Failed to init mem trackers, msg: " << status.get_error_msg();
+ return status;
+ }
+
+ vectorized::Block tmp_block;
+ int result_column = -1;
+ // calc vexpr
+ ctx->execute(&tmp_block, &result_column);
+ DCHECK(result_column != -1);
+ PrimitiveType root_type = ctx->root()->type().type;
+ // covert to thrift type
+ TPrimitiveType::type t_type = doris::to_thrift(root_type);
+
+ // collect result
+ PExprResult expr_result;
+ string result;
+ const auto& column_ptr = tmp_block.get_by_position(result_column).column;
+ if (column_ptr->is_nullable() && column_ptr->is_null_at(0)) {
+ expr_result.set_success(false);
+ } else {
+ expr_result.set_success(true);
+ result = _get_result<true>((void *) column_ptr->get_data_at(0).data, ctx->root()->type().type);
+ }
+
+ expr_result.set_content(std::move(result));
+ expr_result.mutable_type()->set_type(t_type);
+ pexpr_result_map.mutable_map()->insert({n.first, expr_result});
+
+ // close context expr
+ ctx->close(_runtime_state.get());
+ }
+ expr_result_map->insert({m.first, pexpr_result_map});
+ }
+
+ return Status::OK();
+}
+
Status FoldConstantExecutor::_init(const TQueryGlobals& query_globals) {
// init runtime state, runtime profile
TPlanFragmentExecParams params;
@@ -128,11 +191,13 @@ Status FoldConstantExecutor::_init(const TQueryGlobals& query_globals) {
return Status::OK();
}
-Status FoldConstantExecutor::_prepare_and_open(ExprContext* ctx) {
+template <typename Context>
+Status FoldConstantExecutor::_prepare_and_open(Context* ctx) {
ctx->prepare(_runtime_state.get(), RowDescriptor(), _mem_tracker);
return ctx->open(_runtime_state.get());
}
+template <bool is_vec>
string FoldConstantExecutor::_get_result(void* src, PrimitiveType slot_type){
switch (slot_type) {
case TYPE_BOOLEAN: {
@@ -176,10 +241,17 @@ string FoldConstantExecutor::_get_result(void* src, PrimitiveType slot_type){
}
case TYPE_DATE:
case TYPE_DATETIME: {
- const DateTimeValue date_value = *reinterpret_cast<DateTimeValue*>(src);
- char str[MAX_DTVALUE_STR_LEN];
- date_value.to_string(str);
- return str;
+ if constexpr (is_vec) {
+ auto date_value = reinterpret_cast<vectorized::VecDateTimeValue*>(src);
+ char str[MAX_DTVALUE_STR_LEN];
+ date_value->to_string(str);
+ return str;
+ } else {
+ const DateTimeValue date_value = *reinterpret_cast<DateTimeValue *>(src);
+ char str[MAX_DTVALUE_STR_LEN];
+ date_value.to_string(str);
+ return str;
+ }
}
case TYPE_DECIMALV2: {
return reinterpret_cast<DecimalV2Value*>(src)->to_string();
diff --git a/be/src/runtime/fold_constant_executor.h b/be/src/runtime/fold_constant_executor.h
index 93ee988..c7c5a38 100644
--- a/be/src/runtime/fold_constant_executor.h
+++ b/be/src/runtime/fold_constant_executor.h
@@ -36,12 +36,17 @@ class FoldConstantExecutor {
public:
// fold constant expr
Status fold_constant_expr(const TFoldConstantParams& params, PConstantExprResult* response);
+
+ // fold constant vexpr
+ Status fold_constant_vexpr(const TFoldConstantParams& params, PConstantExprResult* response);
private:
// init runtime_state and mem_tracker
Status _init(const TQueryGlobals& query_globals);
// prepare expr
- Status _prepare_and_open(ExprContext* ctx);
+ template <typename Context>
+ Status _prepare_and_open(Context* ctx);
+ template <bool is_vec = false>
std::string _get_result(void* src, PrimitiveType slot_type);
std::unique_ptr<RuntimeState> _runtime_state;
diff --git a/be/src/runtime/mysql_result_writer.cpp b/be/src/runtime/mysql_result_writer.cpp
index 1c2589c..eaf1bd7 100644
--- a/be/src/runtime/mysql_result_writer.cpp
+++ b/be/src/runtime/mysql_result_writer.cpp
@@ -29,6 +29,13 @@
#include "util/mysql_row_buffer.h"
#include "util/types.h"
+#include "vec/core/block.h"
+#include "vec/columns/column_vector.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/common/assert_cast.h"
+#include "vec/exprs/vexpr.h"
+#include "vec/exprs/vexpr_context.h"
+
namespace doris {
MysqlResultWriter::MysqlResultWriter(BufferControlBlock* sinker,
@@ -204,7 +211,6 @@ int MysqlResultWriter::_add_row_value(int index, const TypeDescriptor& type, voi
}
Status MysqlResultWriter::_add_one_row(TupleRow* row) {
- SCOPED_TIMER(_convert_tuple_timer);
_row_buffer->reset();
int num_columns = _output_expr_ctxs.size();
int buf_ret = 0;
diff --git a/be/src/runtime/mysql_result_writer.h b/be/src/runtime/mysql_result_writer.h
index 2754e0c..8b96fec 100644
--- a/be/src/runtime/mysql_result_writer.h
+++ b/be/src/runtime/mysql_result_writer.h
@@ -21,6 +21,8 @@
#include "runtime/result_writer.h"
#include "runtime/runtime_state.h"
+#include "vec/data_types/data_type.h"
+
namespace doris {
class TupleRow;
diff --git a/be/src/runtime/plan_fragment_executor.cpp b/be/src/runtime/plan_fragment_executor.cpp
index a464211..fe64a4f 100644
--- a/be/src/runtime/plan_fragment_executor.cpp
+++ b/be/src/runtime/plan_fragment_executor.cpp
@@ -42,6 +42,10 @@
#include "util/uid_util.h"
#include "util/logging.h"
+#include "vec/core/block.h"
+#include "vec/exec/vexchange_node.h"
+#include "vec/runtime/vdata_stream_mgr.h"
+
namespace doris {
PlanFragmentExecutor::PlanFragmentExecutor(ExecEnv* exec_env,
@@ -159,6 +163,7 @@ Status PlanFragmentExecutor::prepare(const TExecPlanFragmentParams& request,
int num_senders = find_with_default(params.per_exch_num_senders, exch_node->id(), 0);
DCHECK_GT(num_senders, 0);
if (_runtime_state->enable_vectorized_exec()) {
+ static_cast<doris::vectorized::VExchangeNode*>(exch_node)->set_num_senders(num_senders);
} else {
static_cast<ExchangeNode*>(exch_node)->set_num_senders(num_senders);
}
@@ -214,6 +219,7 @@ Status PlanFragmentExecutor::prepare(const TExecPlanFragmentParams& request,
_row_batch.reset(new RowBatch(_plan->row_desc(), _runtime_state->batch_size(),
_runtime_state->instance_mem_tracker().get()));
+ _block.reset(new doris::vectorized::Block());
// _row_batch->tuple_data_pool()->set_limits(*_runtime_state->mem_trackers());
VLOG_NOTICE << "plan_root=\n" << _plan->debug_string();
_prepared = true;
@@ -244,6 +250,7 @@ Status PlanFragmentExecutor::open() {
}
Status status = Status::OK();
if (_runtime_state->enable_vectorized_exec()) {
+ status = open_vectorized_internal();
} else {
status = open_internal();
}
@@ -259,6 +266,82 @@ Status PlanFragmentExecutor::open() {
return status;
}
+Status PlanFragmentExecutor::open_vectorized_internal() {
+ {
+ SCOPED_CPU_TIMER(_fragment_cpu_timer);
+ SCOPED_TIMER(profile()->total_time_counter());
+ RETURN_IF_ERROR(_plan->open(_runtime_state.get()));
+ }
+ if (_sink == nullptr) {
+ return Status::OK();
+ }
+ {
+ SCOPED_CPU_TIMER(_fragment_cpu_timer);
+ RETURN_IF_ERROR(_sink->open(runtime_state()));
+ }
+ doris::vectorized::Block* block = nullptr;
+ while (true) {
+ {
+ SCOPED_CPU_TIMER(_fragment_cpu_timer);
+ RETURN_IF_ERROR(get_vectorized_internal(&block));
+ }
+
+ if (block == NULL) {
+ break;
+ }
+
+ SCOPED_TIMER(profile()->total_time_counter());
+ SCOPED_CPU_TIMER(_fragment_cpu_timer);
+ // Collect this plan and sub plan statistics, and send to parent plan.
+ if (_collect_query_statistics_with_every_batch) {
+ _collect_query_statistics();
+ }
+ RETURN_IF_ERROR(_sink->send(runtime_state(), block));
+ }
+ {
+ SCOPED_TIMER(profile()->total_time_counter());
+ _collect_query_statistics();
+ Status status;
+ {
+ std::lock_guard<std::mutex> l(_status_lock);
+ status = _status;
+ }
+ status = _sink->close(runtime_state(), status);
+ RETURN_IF_ERROR(status);
+ }
+ // Setting to NULL ensures that the d'tor won't double-close the sink.
+ _sink.reset(nullptr);
+ _done = true;
+
+ stop_report_thread();
+ send_report(true);
+
+ return Status::OK();
+}
+Status PlanFragmentExecutor::get_vectorized_internal(::doris::vectorized::Block** block) {
+ if (_done) {
+ *block = nullptr;
+ return Status::OK();
+ }
+
+ auto vexec_node = static_cast<doris::ExecNode*>(_plan);
+ while (!_done) {
+ _block->clear_column_data(vexec_node->row_desc().num_materialized_slots());
+ SCOPED_TIMER(profile()->total_time_counter());
+ RETURN_IF_ERROR(vexec_node->get_next(_runtime_state.get(), _block.get(), &_done));
+
+ if (_block->rows() > 0) {
+ COUNTER_UPDATE(_rows_produced_counter, _block->rows());
+ *block = _block.get();
+ break;
+ }
+
+ *block = nullptr;
+ }
+
+ return Status::OK();
+}
+
Status PlanFragmentExecutor::open_internal() {
{
SCOPED_CPU_TIMER(_fragment_cpu_timer);
@@ -521,8 +604,14 @@ void PlanFragmentExecutor::cancel() {
.query_id(_query_id).instance_id(_runtime_state->fragment_instance_id());
DCHECK(_prepared);
_runtime_state->set_is_cancelled(true);
- _runtime_state->exec_env()->stream_mgr()->cancel(_runtime_state->fragment_instance_id());
- _runtime_state->exec_env()->result_mgr()->cancel(_runtime_state->fragment_instance_id());
+
+ // must close stream_mgr to avoid dead lock in Exchange Node
+ if (_runtime_state->enable_vectorized_exec()) {
+ _runtime_state->exec_env()->vstream_mgr()->cancel(_runtime_state->fragment_instance_id());
+ } else {
+ _runtime_state->exec_env()->stream_mgr()->cancel(_runtime_state->fragment_instance_id());
+ _runtime_state->exec_env()->result_mgr()->cancel(_runtime_state->fragment_instance_id());
+ }
}
void PlanFragmentExecutor::set_abort() {
diff --git a/be/src/runtime/plan_fragment_executor.h b/be/src/runtime/plan_fragment_executor.h
index 1fc4578..3cdb6bb 100644
--- a/be/src/runtime/plan_fragment_executor.h
+++ b/be/src/runtime/plan_fragment_executor.h
@@ -30,11 +30,11 @@
#include "runtime/runtime_state.h"
#include "util/hash_util.hpp"
#include "util/time.h"
+#include "vec/core/block.h"
namespace doris {
class QueryFragmentsCtx;
-class HdfsFsCache;
class ExecNode;
class RowDescriptor;
class RowBatch;
@@ -195,6 +195,7 @@ private:
// Created in prepare (if required), owned by this object.
std::unique_ptr<DataSink> _sink;
std::unique_ptr<RowBatch> _row_batch;
+ std::unique_ptr<doris::vectorized::Block> _block;
// Number of rows returned by this fragment
RuntimeProfile::Counter* _rows_produced_counter;
@@ -234,9 +235,11 @@ private:
// have been closed, a final report will have been sent and the report thread will
// have been stopped. _sink will be set to nullptr after successful execution.
Status open_internal();
+ Status open_vectorized_internal();
// Executes get_next() logic and returns resulting status.
Status get_next_internal(RowBatch** batch);
+ Status get_vectorized_internal(::doris::vectorized::Block** block);
// Stops report thread, if one is running. Blocks until report thread terminates.
// Idempotent.
diff --git a/be/src/runtime/primitive_type.h b/be/src/runtime/primitive_type.h
index 36113a5..224957f 100644
--- a/be/src/runtime/primitive_type.h
+++ b/be/src/runtime/primitive_type.h
@@ -27,6 +27,13 @@
#include "runtime/decimalv2_value.h"
#include "runtime/large_int_value.h"
#include "runtime/string_value.h"
+#include "udf/udf.h"
+
+#include "vec/columns/column_decimal.h"
+#include "vec/columns/column_string.h"
+#include "vec/columns/columns_number.h"
+#include "vec/core/types.h"
+#include "vec/runtime/vdatetime_value.h"
namespace doris {
@@ -60,6 +67,51 @@ enum PrimitiveType {
TYPE_STRING, /* 23 */
};
+inline PrimitiveType convert_type_to_primitive(FunctionContext::Type type) {
+ switch (type) {
+ case FunctionContext::Type::INVALID_TYPE:
+ return PrimitiveType::INVALID_TYPE;
+ case FunctionContext::Type::TYPE_DOUBLE:
+ return PrimitiveType::TYPE_DOUBLE;
+ case FunctionContext::Type::TYPE_NULL:
+ return PrimitiveType::TYPE_NULL;
+ case FunctionContext::Type::TYPE_CHAR:
+ return PrimitiveType::TYPE_CHAR;
+ case FunctionContext::Type::TYPE_VARCHAR:
+ return PrimitiveType::TYPE_VARCHAR;
+ case FunctionContext::Type::TYPE_STRING:
+ return PrimitiveType::TYPE_STRING;
+ case FunctionContext::Type::TYPE_DATETIME:
+ return PrimitiveType::TYPE_DATETIME;
+ case FunctionContext::Type::TYPE_DECIMALV2:
+ return PrimitiveType::TYPE_DECIMALV2;
+ case FunctionContext::Type::TYPE_BOOLEAN:
+ return PrimitiveType::TYPE_BOOLEAN;
+ case FunctionContext::Type::TYPE_ARRAY:
+ return PrimitiveType::TYPE_ARRAY;
+ case FunctionContext::Type::TYPE_OBJECT:
+ return PrimitiveType::TYPE_OBJECT;
+ case FunctionContext::Type::TYPE_HLL:
+ return PrimitiveType::TYPE_HLL;
+ case FunctionContext::Type::TYPE_TINYINT:
+ return PrimitiveType::TYPE_TINYINT;
+ case FunctionContext::Type::TYPE_SMALLINT:
+ return PrimitiveType::TYPE_SMALLINT;
+ case FunctionContext::Type::TYPE_INT:
+ return PrimitiveType::TYPE_INT;
+ case FunctionContext::Type::TYPE_BIGINT:
+ return PrimitiveType::TYPE_BIGINT;
+ case FunctionContext::Type::TYPE_LARGEINT:
+ return PrimitiveType::TYPE_LARGEINT;
+ case FunctionContext::Type::TYPE_DATE:
+ return PrimitiveType::TYPE_DATE;
+ default:
+ DCHECK(false);
+ }
+
+ return PrimitiveType::INVALID_TYPE;
+}
+
inline bool is_enumeration_type(PrimitiveType type) {
switch (type) {
case TYPE_FLOAT:
@@ -98,6 +150,10 @@ inline bool is_string_type(PrimitiveType type) {
return type == TYPE_CHAR || type == TYPE_VARCHAR || type == TYPE_STRING;
}
+inline bool has_variable_type(PrimitiveType type) {
+ return type == TYPE_CHAR || type == TYPE_VARCHAR || type == TYPE_OBJECT || type == TYPE_STRING;
+}
+
// Returns the byte size of 'type' Returns 0 for variable length types.
inline int get_byte_size(PrimitiveType type) {
switch (type) {
@@ -125,9 +181,9 @@ inline int get_byte_size(PrimitiveType type) {
case TYPE_DOUBLE:
return 8;
- case TYPE_LARGEINT:
case TYPE_DATETIME:
case TYPE_DATE:
+ case TYPE_LARGEINT:
case TYPE_DECIMALV2:
return 16;
@@ -220,63 +276,78 @@ struct PrimitiveTypeTraits {};
template <>
struct PrimitiveTypeTraits<TYPE_BOOLEAN> {
using CppType = bool;
+ using ColumnType = vectorized::ColumnUInt8;
};
template <>
struct PrimitiveTypeTraits<TYPE_TINYINT> {
using CppType = int8_t;
+ using ColumnType = vectorized::ColumnInt8;
};
template <>
struct PrimitiveTypeTraits<TYPE_SMALLINT> {
using CppType = int16_t;
+ using ColumnType = vectorized::ColumnInt16;
};
template <>
struct PrimitiveTypeTraits<TYPE_INT> {
using CppType = int32_t;
+ using ColumnType = vectorized::ColumnInt32;
};
template <>
struct PrimitiveTypeTraits<TYPE_BIGINT> {
using CppType = int64_t;
+ using ColumnType = vectorized::ColumnInt64;
};
template <>
struct PrimitiveTypeTraits<TYPE_FLOAT> {
using CppType = float;
+ using ColumnType = vectorized::ColumnFloat32;
};
template <>
struct PrimitiveTypeTraits<TYPE_TIME> {
using CppType = double;
+ using ColumnType = vectorized::ColumnFloat64;
};
template <>
struct PrimitiveTypeTraits<TYPE_DOUBLE> {
using CppType = double;
+ using ColumnType = vectorized::ColumnFloat64;
};
template <>
struct PrimitiveTypeTraits<TYPE_DATE> {
- using CppType = DateTimeValue;
+ using CppType = doris::DateTimeValue;
+ using ColumnType = vectorized::ColumnVector<vectorized::DateTime>;
};
template <>
struct PrimitiveTypeTraits<TYPE_DATETIME> {
- using CppType = DateTimeValue;
+ using CppType = doris::DateTimeValue;
+ using ColumnType = vectorized::ColumnVector<vectorized::DateTime>;
};
template <>
struct PrimitiveTypeTraits<TYPE_DECIMALV2> {
using CppType = DecimalV2Value;
+ using ColumnType = vectorized::ColumnDecimal<vectorized::Decimal128>;
};
template <>
struct PrimitiveTypeTraits<TYPE_LARGEINT> {
using CppType = __int128_t;
+ using ColumnType = vectorized::ColumnInt128;
};
template <>
struct PrimitiveTypeTraits<TYPE_CHAR> {
using CppType = StringValue;
+ using ColumnType = vectorized::ColumnString;
};
template <>
struct PrimitiveTypeTraits<TYPE_VARCHAR> {
using CppType = StringValue;
+ using ColumnType = vectorized::ColumnString;
};
template <>
struct PrimitiveTypeTraits<TYPE_STRING> {
using CppType = StringValue;
+ using ColumnType = vectorized::ColumnString;
};
} // namespace doris
diff --git a/be/src/runtime/raw_value.h b/be/src/runtime/raw_value.h
index e9e270e..e16bbf5 100644
--- a/be/src/runtime/raw_value.h
+++ b/be/src/runtime/raw_value.h
@@ -87,6 +87,9 @@ public:
// TODO: fix get_hash_value
static uint32_t zlib_crc32(const void* value, const TypeDescriptor& type, uint32_t seed);
+ // Same as the up function, only use in vec exec engine.
+ static uint32_t zlib_crc32(const void* value, size_t len, const TypeDescriptor& type, uint32_t seed);
+
// Compares both values.
// Return value is < 0 if v1 < v2, 0 if v1 == v2, > 0 if v1 > v2.
static int compare(const void* v1, const void* v2, const TypeDescriptor& type);
@@ -399,6 +402,59 @@ inline uint32_t RawValue::zlib_crc32(const void* v, const TypeDescriptor& type,
}
}
+// NOTE: this is just for split data, decimal use old doris hash function
+// Because crc32 hardware is not equal with zlib crc32
+inline uint32_t RawValue::zlib_crc32(const void* v, size_t len, const TypeDescriptor& type, uint32_t seed) {
+ // Hash_combine with v = 0
+ if (v == nullptr) {
+ uint32_t value = 0x9e3779b9;
+ return seed ^ (value + (seed << 6) + (seed >> 2));
+ }
+
+ switch (type.type) {
+ case TYPE_VARCHAR:
+ case TYPE_HLL:
+ case TYPE_STRING:
+ case TYPE_CHAR: {
+ return HashUtil::zlib_crc_hash(v, len, seed);
+ }
+
+ case TYPE_BOOLEAN:
+ case TYPE_TINYINT:
+ return HashUtil::zlib_crc_hash(v, 1, seed);
+ case TYPE_SMALLINT:
+ return HashUtil::zlib_crc_hash(v, 2, seed);
+ case TYPE_INT:
+ return HashUtil::zlib_crc_hash(v, 4, seed);
+ case TYPE_BIGINT:
+ return HashUtil::zlib_crc_hash(v, 8, seed);
+ case TYPE_LARGEINT:
+ return HashUtil::zlib_crc_hash(v, 16, seed);
+ case TYPE_FLOAT:
+ return HashUtil::zlib_crc_hash(v, 4, seed);
+ case TYPE_DOUBLE:
+ return HashUtil::zlib_crc_hash(v, 8, seed);
+ case TYPE_DATE:
+ case TYPE_DATETIME: {
+ auto* date_val = (const vectorized::VecDateTimeValue*)v;
+ char buf[64];
+ int len = date_val->to_buffer(buf);
+ return HashUtil::zlib_crc_hash(buf, len, seed);
+ }
+
+ case TYPE_DECIMALV2: {
+ const DecimalV2Value* dec_val = (const DecimalV2Value*)v;
+ int64_t int_val = dec_val->int_value();
+ int32_t frac_val = dec_val->frac_value();
+ seed = HashUtil::zlib_crc_hash(&int_val, sizeof(int_val), seed);
+ return HashUtil::zlib_crc_hash(&frac_val, sizeof(frac_val), seed);
+ }
+ default:
+ DCHECK(false) << "invalid type: " << type;
+ return 0;
+ }
+}
+
} // namespace doris
#endif
diff --git a/be/src/runtime/result_sink.cpp b/be/src/runtime/result_sink.cpp
index 206b0f0..60538ca 100644
--- a/be/src/runtime/result_sink.cpp
+++ b/be/src/runtime/result_sink.cpp
@@ -29,6 +29,8 @@
#include "runtime/runtime_state.h"
#include "util/uid_util.h"
+#include "vec/exprs/vexpr.h"
+
namespace doris {
ResultSink::ResultSink(const RowDescriptor& row_desc, const std::vector<TExpr>& t_output_expr,
diff --git a/be/src/runtime/row_batch.cpp b/be/src/runtime/row_batch.cpp
index 85ba3c7..3e7914e 100644
--- a/be/src/runtime/row_batch.cpp
+++ b/be/src/runtime/row_batch.cpp
@@ -30,8 +30,8 @@
#include "runtime/string_value.h"
#include "runtime/tuple_row.h"
-//#include "vec/columns/column_vector.h"
-//#include "vec/core/block.h"
+#include "vec/columns/column_vector.h"
+#include "vec/core/block.h"
using std::vector;
@@ -625,6 +625,57 @@ void RowBatch::transfer_resource_ownership(RowBatch* dest) {
reset();
}
+vectorized::Block RowBatch::convert_to_vec_block() const {
+ std::vector<vectorized::MutableColumnPtr> columns;
+ for (const auto tuple_desc : _row_desc.tuple_descriptors()) {
+ for (const auto slot_desc : tuple_desc->slots()) {
+ columns.emplace_back(slot_desc->get_empty_mutable_column());
+ }
+ }
+
+ std::vector<SlotDescriptor*> slot_descs;
+ std::vector<int> tuple_idx;
+ int column_numbers = 0;
+ for (int i = 0; i < _row_desc.tuple_descriptors().size(); ++i) {
+ auto tuple_desc = _row_desc.tuple_descriptors()[i];
+ for (int j = 0; j < tuple_desc->slots().size(); ++j) {
+ slot_descs.push_back(tuple_desc->slots()[j]);
+ tuple_idx.push_back(i);
+ }
+ column_numbers += tuple_desc->slots().size();
+ }
+ for (int i = 0; i < column_numbers; ++i) {
+ auto slot_desc = slot_descs[i];
+ for (int j = 0; j < _num_rows; ++j) {
+ TupleRow* src_row = get_row(j);
+ auto tuple = src_row->get_tuple(tuple_idx[i]);
+ if (slot_desc->is_nullable() && tuple->is_null(slot_desc->null_indicator_offset())) {
+ columns[i]->insert_data(nullptr, 0);
+ } else if (slot_desc->type().is_string_type()) {
+ auto string_value =
+ static_cast<const StringValue*>(tuple->get_slot(slot_desc->tuple_offset()));
+ columns[i]->insert_data(string_value->ptr, string_value->len);
+ } else {
+ columns[i]->insert_data(
+ static_cast<const char*>(tuple->get_slot(slot_desc->tuple_offset())),
+ slot_desc->slot_size());
+ }
+ }
+ }
+
+ doris::vectorized::ColumnsWithTypeAndName columns_with_type_and_name;
+ auto n_columns = 0;
+ for (const auto tuple_desc : _row_desc.tuple_descriptors()) {
+ for (const auto slot_desc : tuple_desc->slots()) {
+ columns_with_type_and_name.emplace_back(columns[n_columns++]->get_ptr(),
+ slot_desc->get_data_type_ptr(),
+ slot_desc->col_name());
+ }
+ }
+
+ return {columns_with_type_and_name};
+}
+
size_t RowBatch::get_batch_size(const TRowBatch& batch) {
size_t result = batch.tuple_data.size();
result += batch.row_tuples.size() * sizeof(TTupleId);
@@ -688,6 +739,7 @@ void RowBatch::deep_copy_to(RowBatch* dst) {
}
dst->commit_rows(_num_rows);
}
+
// TODO: consider computing size of batches as they are built up
size_t RowBatch::total_byte_size() const {
size_t result = 0;
diff --git a/be/src/runtime/row_batch.h b/be/src/runtime/row_batch.h
index 596fa70..4333c73 100644
--- a/be/src/runtime/row_batch.h
+++ b/be/src/runtime/row_batch.h
@@ -43,6 +43,7 @@ class TupleRow;
class TupleDescriptor;
class PRowBatch;
+
// A RowBatch encapsulates a batch of rows, each composed of a number of tuples.
// The maximum number of rows is fixed at the time of construction, and the caller
// can add rows up to that capacity.
@@ -315,7 +316,7 @@ public:
// we firstly update dest resource, and then reset current resource
void transfer_resource_ownership(RowBatch* dest);
- void copy_row(const TupleRow* src, TupleRow* dest) const {
+ void copy_row(TupleRow* src, TupleRow* dest) {
memcpy(dest, src, _num_tuples_per_row * sizeof(Tuple*));
}
@@ -362,6 +363,8 @@ public:
static size_t get_batch_size(const TRowBatch& batch);
static size_t get_batch_size(const PRowBatch& batch);
+ vectorized::Block convert_to_vec_block() const;
+
int num_rows() const { return _num_rows; }
int capacity() const { return _capacity; }
diff --git a/be/src/runtime/string_value.hpp b/be/src/runtime/string_value.hpp
index dd29bff..c44115d 100644
--- a/be/src/runtime/string_value.hpp
+++ b/be/src/runtime/string_value.hpp
@@ -42,22 +42,19 @@ static inline int string_compare(const char* s1, int64_t n1, const char* s2, int
int64_t len) {
DCHECK_EQ(len, std::min(n1, n2));
#ifdef __SSE4_2__
- if (CpuInfo::is_supported(CpuInfo::SSE4_2)) {
- while (len >= sse_util::CHARS_PER_128_BIT_REGISTER) {
- __m128i xmm0 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(s1));
- __m128i xmm1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(s2));
- int chars_match =
- _mm_cmpestri(xmm0, sse_util::CHARS_PER_128_BIT_REGISTER, xmm1,
- sse_util::CHARS_PER_128_BIT_REGISTER, sse_util::STRCMP_MODE);
- if (chars_match != sse_util::CHARS_PER_128_BIT_REGISTER) {
- return (unsigned char)s1[chars_match] - (unsigned char)s2[chars_match];
- }
- len -= sse_util::CHARS_PER_128_BIT_REGISTER;
- s1 += sse_util::CHARS_PER_128_BIT_REGISTER;
- s2 += sse_util::CHARS_PER_128_BIT_REGISTER;
+ while (len >= sse_util::CHARS_PER_128_BIT_REGISTER) {
+ __m128i xmm0 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(s1));
+ __m128i xmm1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(s2));
+ int chars_match =
+ _mm_cmpestri(xmm0, sse_util::CHARS_PER_128_BIT_REGISTER, xmm1,
+ sse_util::CHARS_PER_128_BIT_REGISTER, sse_util::STRCMP_MODE);
+ if (chars_match != sse_util::CHARS_PER_128_BIT_REGISTER) {
+ return (unsigned char)s1[chars_match] - (unsigned char)s2[chars_match];
}
+ len -= sse_util::CHARS_PER_128_BIT_REGISTER;
+ s1 += sse_util::CHARS_PER_128_BIT_REGISTER;
+ s2 += sse_util::CHARS_PER_128_BIT_REGISTER;
}
-
#endif
unsigned char u1, u2;
while (len-- > 0) {
diff --git a/be/src/runtime/types.h b/be/src/runtime/types.h
index e40bc5b..c2adca9 100644
--- a/be/src/runtime/types.h
+++ b/be/src/runtime/types.h
@@ -28,6 +28,12 @@
#include "runtime/collection_value.h"
#include "runtime/primitive_type.h"
#include "thrift/protocol/TDebugProtocol.h"
+#include "vec/data_types/data_type_bitmap.h"
+#include "vec/data_types/data_type_date.h"
+#include "vec/data_types/data_type_date_time.h"
+#include "vec/data_types/data_type_decimal.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
namespace doris {
@@ -272,6 +278,58 @@ struct TypeDescriptor {
return -1;
}
+ inline doris::vectorized::DataTypePtr get_data_type_ptr() const {
+ switch (type) {
+ case TYPE_BOOLEAN:
+ return std::make_shared<vectorized::DataTypeUInt8>();
+
+ case TYPE_TINYINT:
+ return std::make_shared<vectorized::DataTypeInt8>();
+
+ case TYPE_SMALLINT:
+ return std::make_shared<vectorized::DataTypeInt16>();
+
+ case TYPE_INT:
+ return std::make_shared<vectorized::DataTypeInt32>();
+
+ case TYPE_FLOAT:
+ return std::make_shared<vectorized::DataTypeFloat32>();
+
+ case TYPE_BIGINT:
+ return std::make_shared<vectorized::DataTypeInt64>();
+
+ case TYPE_LARGEINT:
+ return std::make_shared<vectorized::DataTypeInt128>();
+ case TYPE_DATE:
+ return std::make_shared<vectorized::DataTypeDate>();
+ case TYPE_DATETIME:
+ return std::make_shared<vectorized::DataTypeDateTime>();
+ case TYPE_TIME:
+ case TYPE_DOUBLE:
+ return std::make_shared<vectorized::DataTypeFloat64>();
+
+ case TYPE_STRING:
+ case TYPE_CHAR:
+ case TYPE_VARCHAR:
+ case TYPE_HLL:
+ return std::make_shared<vectorized::DataTypeString>();
+ case TYPE_OBJECT:
+ return std::make_shared<vectorized::DataTypeBitMap>();
+
+ case TYPE_DECIMALV2:
+ return std::make_shared<vectorized::DataTypeDecimal<vectorized::Decimal128>>(27, 9);
+ // Just Mock A NULL Type in Vec Exec Engine
+ case TYPE_NULL:
+ return std::make_shared<vectorized::DataTypeUInt8>();
+
+ case INVALID_TYPE:
+ default:
+ DCHECK(false);
+ }
+ // For llvm complain
+ return nullptr;
+ }
+
static inline int get_decimal_byte_size(int precision) {
DCHECK_GT(precision, 0);
if (precision <= MAX_DECIMAL4_PRECISION) {
diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp
index ee6cf7e..4106963 100644
--- a/be/src/service/internal_service.cpp
+++ b/be/src/service/internal_service.cpp
@@ -36,6 +36,7 @@
#include "util/string_util.h"
#include "util/thrift_util.h"
#include "util/uid_util.h"
+#include "vec/runtime/vdata_stream_mgr.h"
namespace doris {
@@ -422,7 +423,10 @@ Status PInternalServiceImpl<T>::_fold_constant_expr(const std::string& ser_reque
uint32_t len = ser_request.size();
RETURN_IF_ERROR(deserialize_thrift_msg(buf, &len, false, &t_request));
}
- return FoldConstantExecutor().fold_constant_expr(t_request, response);
+ if (!t_request.__isset.vec_exec || !t_request.vec_exec)
+ return FoldConstantExecutor().fold_constant_expr(t_request, response);
+
+ return FoldConstantExecutor().fold_constant_vexpr(t_request, response);
}
template <typename T>
@@ -432,6 +436,7 @@ void PInternalServiceImpl<T>::transmit_block(google::protobuf::RpcController* cn
google::protobuf::Closure* done) {
VLOG_ROW << "transmit data: fragment_instance_id=" << print_id(request->finst_id())
<< " node=" << request->node_id();
+ _exec_env->vstream_mgr()->transmit_block(request, &done);
if (done != nullptr) {
done->Run();
}
diff --git a/be/src/udf/udf.cpp b/be/src/udf/udf.cpp
index 0360f39..269bd88 100644
--- a/be/src/udf/udf.cpp
+++ b/be/src/udf/udf.cpp
@@ -131,6 +131,10 @@ void FunctionContextImpl::set_constant_args(const std::vector<doris_udf::AnyVal*
_constant_args = constant_args;
}
+void FunctionContextImpl::set_constant_cols(const std::vector<doris::ColumnPtrWrapper*>& constant_cols) {
+ _constant_cols = constant_cols;
+}
+
bool FunctionContextImpl::check_allocations_empty() {
if (_allocations.empty() && _external_bytes_tracked == 0) {
return true;
@@ -187,6 +191,7 @@ FunctionContext* FunctionContextImpl::clone(MemPool* pool) {
create_context(_state, pool, _intermediate_type, _return_type, _arg_types,
_varargs_buffer_size, _debug);
new_context->_impl->_constant_args = _constant_args;
+ new_context->_impl->_constant_cols = _constant_cols;
new_context->_impl->_fragment_local_fn_state = _fragment_local_fn_state;
return new_context;
}
diff --git a/be/src/udf/udf.h b/be/src/udf/udf.h
index 3b9ff5a..141219a 100644
--- a/be/src/udf/udf.h
+++ b/be/src/udf/udf.h
@@ -28,6 +28,7 @@
// object serves as the interface object between the UDF/UDA and the doris process.
namespace doris {
class FunctionContextImpl;
+class ColumnPtrWrapper;
struct StringValue;
struct BitmapValue;
struct DecimalV2Value;
@@ -224,12 +225,16 @@ public:
// FunctionContext* argument) is a constant (e.g. 5, "string", 1 + 1).
bool is_arg_constant(int arg_idx) const;
+ bool is_col_constant(int arg_idx) const;
+
// Returns a pointer to the value of the arg_idx-th input argument (0 indexed, not
// including the FunctionContext* argument). Returns nullptr if the argument is not
// constant. This function can be used to obtain user-specified constants in a UDF's
// Init() or Close() functions.
AnyVal* get_constant_arg(int arg_idx) const;
+ doris::ColumnPtrWrapper* get_constant_col(int arg_idx) const;
+
// Create a test FunctionContext object. The caller is responsible for calling delete
// on it. This context has additional debugging validation enabled.
static FunctionContext* create_test_context();
diff --git a/be/src/udf/udf_internal.h b/be/src/udf/udf_internal.h
index 2d2c318..085002d 100644
--- a/be/src/udf/udf_internal.h
+++ b/be/src/udf/udf_internal.h
@@ -32,6 +32,7 @@ namespace doris {
class FreePool;
class MemPool;
class RuntimeState;
+class ColumnPtrWrapper;
// This class actually implements the interface of FunctionContext. This is split to
// hide the details from the external header.
@@ -67,6 +68,8 @@ public:
void set_constant_args(const std::vector<doris_udf::AnyVal*>& constant_args);
+ void set_constant_cols(const std::vector<doris::ColumnPtrWrapper*>& cols);
+
uint8_t* varargs_buffer() { return _varargs_buffer; }
std::vector<doris_udf::AnyVal*>* staging_input_vals() { return &_staging_input_vals; }
@@ -169,6 +172,8 @@ private:
// value of the argument.
std::vector<doris_udf::AnyVal*> _constant_args;
+ std::vector<doris::ColumnPtrWrapper*> _constant_cols;
+
// Used by ScalarFnCall to store the arguments when running without codegen. Allows us
// to pass AnyVal* arguments to the scalar function directly, rather than codegening a
// call that passes the correct AnyVal subclass pointer type.
diff --git a/be/src/udf/udf_ir.cpp b/be/src/udf/udf_ir.cpp
index 4eefd17..a1ceb02 100644
--- a/be/src/udf/udf_ir.cpp
+++ b/be/src/udf/udf_ir.cpp
@@ -25,6 +25,13 @@ bool FunctionContext::is_arg_constant(int i) const {
return _impl->_constant_args[i] != nullptr;
}
+bool FunctionContext::is_col_constant(int i) const {
+ if (i < 0 || i >= _impl->_constant_cols.size()) {
+ return false;
+ }
+ return _impl->_constant_cols[i] != nullptr;
+}
+
AnyVal* FunctionContext::get_constant_arg(int i) const {
if (i < 0 || i >= _impl->_constant_args.size()) {
return nullptr;
@@ -32,6 +39,13 @@ AnyVal* FunctionContext::get_constant_arg(int i) const {
return _impl->_constant_args[i];
}
+doris::ColumnPtrWrapper* FunctionContext::get_constant_col(int i) const {
+ if (i < 0 || i >= _impl->_constant_cols.size()) {
+ return nullptr;
+ }
+ return _impl->_constant_cols[i];
+}
+
int FunctionContext::get_num_args() const {
return _impl->_arg_types.size();
}
diff --git a/be/src/util/binary_cast.hpp b/be/src/util/binary_cast.hpp
index d528d71..764f5b5 100644
--- a/be/src/util/binary_cast.hpp
+++ b/be/src/util/binary_cast.hpp
@@ -23,7 +23,7 @@
#include "runtime/datetime_value.h"
#include "runtime/decimalv2_value.h"
#include "util/types.h"
-
+#include "vec/runtime/vdatetime_value.h"
namespace doris {
union TypeConverter {
uint64_t u64;
@@ -56,6 +56,11 @@ union DateTimeInt128Union {
~DateTimeInt128Union() {}
};
+union VecDateTimeInt64Union {
+ doris::vectorized::VecDateTimeValue dt;
+ __int64_t i64;
+ ~VecDateTimeInt64Union() {}
+};
// similar to reinterpret_cast but won't break strict-aliasing rules
template <typename From, typename To>
To binary_cast(From from) {
@@ -66,11 +71,13 @@ To binary_cast(From from) {
constexpr bool from_decv2_to_packed128 = match_v<From, DecimalV2Value, To, PackedInt128>;
constexpr bool from_i128_to_dt = match_v<From, __int128_t, To, DateTimeValue>;
constexpr bool from_dt_to_i128 = match_v<From, DateTimeValue, To, __int128_t>;
+ constexpr bool from_i64_to_vec_dt = match_v<From, __int64_t, To, doris::vectorized::VecDateTimeValue>;
+ constexpr bool from_vec_dt_to_i64 = match_v<From, doris::vectorized::VecDateTimeValue, To, __int64_t>;
constexpr bool from_i128_to_decv2 = match_v<From, __int128_t, To, DecimalV2Value>;
constexpr bool from_decv2_to_i128 = match_v<From, DecimalV2Value, To, __int128_t>;
static_assert(from_u64_to_db || from_i64_to_db || from_db_to_i64 || from_db_to_u64 ||
- from_decv2_to_packed128 || from_i128_to_dt || from_dt_to_i128 ||
+ from_decv2_to_packed128 || from_i128_to_dt || from_dt_to_i128 || from_i64_to_vec_dt || from_vec_dt_to_i64 ||
from_i128_to_decv2 || from_decv2_to_i128);
if constexpr (from_u64_to_db) {
@@ -99,6 +106,12 @@ To binary_cast(From from) {
} else if constexpr (from_dt_to_i128) {
DateTimeInt128Union conv = {.dt = from};
return conv.i128;
+ } else if constexpr (from_i64_to_vec_dt) {
+ VecDateTimeInt64Union conv = {.i64 = from};
+ return conv.dt;
+ } else if constexpr (from_vec_dt_to_i64) {
+ VecDateTimeInt64Union conv = {.dt = from};
+ return conv.i64;
} else if constexpr (from_i128_to_decv2) {
DecimalInt128Union conv;
conv.i128 = from;
diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h
index 81fb10b..bdb124b 100644
--- a/be/src/util/bitmap_value.h
+++ b/be/src/util/bitmap_value.h
@@ -1691,6 +1691,12 @@ public:
}
return count;
}
+
+ void clear() {
+ _type = EMPTY;
+ _bitmap.clear();
+ _sv = 0;
+ }
// Implement an iterator for convenience
friend class BitmapValueIterator;
diff --git a/be/src/util/brpc_stub_cache.h b/be/src/util/brpc_stub_cache.h
index 53ee3b7..e944aac 100644
--- a/be/src/util/brpc_stub_cache.h
+++ b/be/src/util/brpc_stub_cache.h
@@ -66,7 +66,7 @@ public:
return stub;
}
- inline std::shared_ptr<PBackendService_Stub> get_stub(const TNetworkAddress& taddr) {
+ virtual std::shared_ptr<PBackendService_Stub> get_stub(const TNetworkAddress& taddr) {
butil::EndPoint endpoint;
if (str2endpoint(taddr.hostname.c_str(), taddr.port, &endpoint)) {
LOG(WARNING) << "unknown endpoint, hostname=" << taddr.hostname
diff --git a/be/src/util/runtime_profile.h b/be/src/util/runtime_profile.h
index f2632d5..1e9366f 100644
--- a/be/src/util/runtime_profile.h
+++ b/be/src/util/runtime_profile.h
@@ -52,8 +52,9 @@ namespace doris {
ScopedTimer<ThreadCpuStopWatch> MACRO_CONCAT(SCOPED_TIMER, __COUNTER__)(c)
#define CANCEL_SAFE_SCOPED_TIMER(c, is_cancelled) \
ScopedTimer<MonotonicStopWatch> MACRO_CONCAT(SCOPED_TIMER, __COUNTER__)(c, is_cancelled)
-#define SCOPED_RAW_TIMER(c) \
- ScopedRawTimer<MonotonicStopWatch, int64_t> MACRO_CONCAT(SCOPED_RAW_TIMER, __COUNTER__)(c)
+#define SCOPED_RAW_TIMER(c) \
+ doris::ScopedRawTimer<doris::MonotonicStopWatch, int64_t> MACRO_CONCAT(SCOPED_RAW_TIMER, \
+ __COUNTER__)(c)
#define SCOPED_ATOMIC_TIMER(c) \
ScopedRawTimer<MonotonicStopWatch, std::atomic<int64_t>> MACRO_CONCAT(SCOPED_ATOMIC_TIMER, \
__COUNTER__)(c)
diff --git a/be/src/util/static_asserts.cpp b/be/src/util/static_asserts.cpp
index eb50636..43833ad 100644
--- a/be/src/util/static_asserts.cpp
+++ b/be/src/util/static_asserts.cpp
@@ -17,7 +17,7 @@
#include "runtime/datetime_value.h"
#include "runtime/string_value.h"
-
+#include "vec/runtime/vdatetime_value.h"
namespace doris {
// This class is unused. It contains static (compile time) asserts.
// This is useful to validate struct sizes and other similar things
@@ -28,6 +28,7 @@ private:
static_assert(offsetof(StringValue, len) == 8);
// Datetime value
static_assert(sizeof(DateTimeValue) == 16);
+ static_assert(sizeof(doris::vectorized::VecDateTimeValue) == 8);
// static_assert(offsetof(DateTimeValue, _year) == 8);
};
diff --git a/be/src/util/url_coding.cpp b/be/src/util/url_coding.cpp
index 862c34cc..3c41114 100644
--- a/be/src/util/url_coding.cpp
+++ b/be/src/util/url_coding.cpp
@@ -194,6 +194,10 @@ int64_t base64_decode(const char* data, size_t length, char* decoded_data) {
// run through the whole string, converting as we go
while ((ch = *current++) != '\0' && length-- > 0) {
+ if (ch >= 256 || ch < 0) {
+ return -1;
+ }
+
if (ch == base64_pad) {
if (*current != '=' && (i % 4) == 1) {
return -1;
diff --git a/be/src/vec/CMakeLists.txt b/be/src/vec/CMakeLists.txt
new file mode 100644
index 0000000..1738819
--- /dev/null
+++ b/be/src/vec/CMakeLists.txt
@@ -0,0 +1,153 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# where to put generated libraries
+set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/vec")
+# where to put generated binaries
+set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/vec")
+
+set(VEC_FILES
+ aggregate_functions/aggregate_function_avg.cpp
+ aggregate_functions/aggregate_function_count.cpp
+ aggregate_functions/aggregate_function_distinct.cpp
+ aggregate_functions/aggregate_function_sum.cpp
+ aggregate_functions/aggregate_function_min_max.cpp
+ aggregate_functions/aggregate_function_null.cpp
+ aggregate_functions/aggregate_function_uniq.cpp
+ aggregate_functions/aggregate_function_hll_union_agg.cpp
+ aggregate_functions/aggregate_function_bitmap.cpp
+ aggregate_functions/aggregate_function_reader.cpp
+ aggregate_functions/aggregate_function_window.cpp
+ aggregate_functions/aggregate_function_simple_factory.cpp
+ columns/collator.cpp
+ columns/column.cpp
+ columns/column_const.cpp
+ columns/column_decimal.cpp
+ columns/column_nullable.cpp
+ columns/column_string.cpp
+ columns/column_vector.cpp
+ columns/columns_common.cpp
+ common/demangle.cpp
+ common/exception.cpp
+ common/pod_array.cpp
+ common/string_utils/string_utils.cpp
+ core/block.cpp
+ core/block_info.cpp
+ core/column_with_type_and_name.cpp
+ core/field.cpp
+ core/field.cpp
+ core/sort_block.cpp
+ core/materialize_block.cpp
+ data_types/data_type.cpp
+ data_types/data_type_bitmap.cpp
+ data_types/data_type_nothing.cpp
+ data_types/data_type_nothing.cpp
+ data_types/data_type_nullable.cpp
+ data_types/data_type_nullable.cpp
+ data_types/data_type_number_base.cpp
+ data_types/data_type_string.cpp
+ data_types/data_type_decimal.cpp
+ data_types/get_least_supertype.cpp
+ data_types/nested_utils.cpp
+ data_types/data_type_date.cpp
+ data_types/data_type_date_time.cpp
+ exec/vaggregation_node.cpp
+ exec/ves_http_scan_node.cpp
+ exec/ves_http_scanner.cpp
+ exec/volap_scan_node.cpp
+ exec/vsort_node.cpp
+ exec/vsort_exec_exprs.cpp
+ exec/volap_scanner.cpp
+ exec/vexchange_node.cpp
+ exec/vset_operation_node.cpp
+ exec/vunion_node.cpp
+ exec/vintersect_node.cpp
+ exec/vexcept_node.cpp
+ exec/vselect_node.cpp
+ exec/vblocking_join_node.cpp
+ exec/vcross_join_node.cpp
+ exec/vodbc_scan_node.cpp
+ exec/vmysql_scan_node.cpp
+ exec/vschema_scan_node.cpp
+ exec/vempty_set_node.cpp
+ exec/vanalytic_eval_node.cpp
+ exec/vassert_num_rows_node.cpp
+ exec/join/vhash_join_node.cpp
+ exprs/vectorized_agg_fn.cpp
+ exprs/vectorized_fn_call.cpp
+ exprs/vexpr.cpp
+ exprs/vexpr_context.cpp
+ exprs/vliteral.cpp
+ exprs/vin_predicate.cpp
+ exprs/vslot_ref.cpp
+ exprs/vcast_expr.cpp
+ exprs/vcase_expr.cpp
+ exprs/vinfo_func.cpp
+ functions/math.cpp
+ functions/function_bitmap.cpp
+ functions/comparison.cpp
+ functions/comparison_less.cpp
+ functions/comparison_equals.cpp
+ functions/comparison_greater.cpp
+ functions/function.cpp
+ functions/function_helpers.cpp
+ functions/function_hash.cpp
+ functions/functions_logical.cpp
+ functions/function_case.cpp
+ functions/function_cast.cpp
+ functions/function_string.cpp
+ functions/function_timestamp.cpp
+ functions/comparison_equal_for_null.cpp
+ functions/function_json.cpp
+ functions/hll_cardinality.cpp
+ functions/hll_empty.cpp
+ functions/hll_hash.cpp
+ functions/plus.cpp
+ functions/modulo.cpp
+ functions/multiply.cpp
+ functions/minus.cpp
+ functions/int_div.cpp
+ functions/divide.cpp
+ functions/function_bit.cpp
+ functions/is_null.cpp
+ functions/is_not_null.cpp
+ functions/in.cpp
+ functions/like.cpp
+ functions/to_time_function.cpp
+ functions/time_of_function.cpp
+ functions/if.cpp
+ functions/function_ifnull.cpp
+ functions/nullif.cpp
+ functions/random.cpp
+ functions/function_date_or_datetime_computation.cpp
+ functions/function_date_or_datetime_to_string.cpp
+ functions/function_datetime_string_to_string.cpp
+ olap/vgeneric_iterators.cpp
+ olap/vcollect_iterator.cpp
+ olap/block_reader.cpp
+ sink/mysql_result_writer.cpp
+ sink/result_sink.cpp
+ sink/vdata_stream_sender.cpp
+ sink/vtabet_sink.cpp
+ runtime/vdatetime_value.cpp
+ runtime/vdata_stream_recvr.cpp
+ runtime/vdata_stream_mgr.cpp
+ runtime/vpartition_info.cpp
+ runtime/vsorted_run_merger.cpp)
+
+add_library(Vec STATIC
+ ${VEC_FILES}
+)
diff --git a/be/src/vec/aggregate_functions/aggregate_function.h b/be/src/vec/aggregate_functions/aggregate_function.h
new file mode 100644
index 0000000..412382f
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function.h
@@ -0,0 +1,237 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/AggregateFunctions/IAggregateFunction.h
+// and modified by Doris
+
+#pragma once
+
+#include <cstddef>
+#include <istream>
+#include <memory>
+#include <ostream>
+#include <type_traits>
+#include <vector>
+
+#include "vec/common/exception.h"
+#include "vec/core/block.h"
+#include "vec/core/column_numbers.h"
+#include "vec/core/field.h"
+#include "vec/core/types.h"
+
+namespace doris::vectorized {
+
+class Arena;
+class IColumn;
+class IDataType;
+
+using DataTypePtr = std::shared_ptr<const IDataType>;
+using DataTypes = std::vector<DataTypePtr>;
+
+using AggregateDataPtr = char*;
+using ConstAggregateDataPtr = const char*;
+
+/** Aggregate functions interface.
+ * Instances of classes with this interface do not contain the data itself for aggregation,
+ * but contain only metadata (description) of the aggregate function,
+ * as well as methods for creating, deleting and working with data.
+ * The data resulting from the aggregation (intermediate computing states) is stored in other objects
+ * (which can be created in some memory pool),
+ * and IAggregateFunction is the external interface for manipulating them.
+ */
+class IAggregateFunction {
+public:
+ IAggregateFunction(const DataTypes& argument_types_, const Array& parameters_)
+ : argument_types(argument_types_), parameters(parameters_) {}
+
+ /// Get main function name.
+ virtual String get_name() const = 0;
+
+ /// Get the result type.
+ virtual DataTypePtr get_return_type() const = 0;
+
+ virtual ~IAggregateFunction() {}
+
+ /** Create empty data for aggregation with `placement new` at the specified location.
+ * You will have to destroy them using the `destroy` method.
+ */
+ virtual void create(AggregateDataPtr __restrict place) const = 0;
+
+ /// Delete data for aggregation.
+ virtual void destroy(AggregateDataPtr __restrict place) const noexcept = 0;
+
+ /// Reset aggregation state
+ virtual void reset(AggregateDataPtr place) const = 0;
+
+ /// It is not necessary to delete data.
+ virtual bool has_trivial_destructor() const = 0;
+
+ /// Get `sizeof` of structure with data.
+ virtual size_t size_of_data() const = 0;
+
+ /// How the data structure should be aligned. NOTE: Currently not used (structures with aggregation state are put without alignment).
+ virtual size_t align_of_data() const = 0;
+
+ /** Adds a value into aggregation data on which place points to.
+ * columns points to columns containing arguments of aggregation function.
+ * row_num is number of row which should be added.
+ * Additional parameter arena should be used instead of standard memory allocator if the addition requires memory allocation.
+ */
+ virtual void add(AggregateDataPtr __restrict place, const IColumn** columns, size_t row_num,
+ Arena* arena) const = 0;
+
+ /// Merges state (on which place points to) with other state of current aggregation function.
+ virtual void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena* arena) const = 0;
+
+ /// Serializes state (to transmit it over the network, for example).
+ virtual void serialize(ConstAggregateDataPtr __restrict place, BufferWritable& buf) const = 0;
+
+ /// Deserializes state. This function is called only for empty (just created) states.
+ virtual void deserialize(AggregateDataPtr __restrict place, BufferReadable& buf, Arena* arena) const = 0;
+
+ /// Returns true if a function requires Arena to handle own states (see add(), merge(), deserialize()).
+ virtual bool allocates_memory_in_arena() const { return false; }
+
+ /// Inserts results into a column.
+ virtual void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const = 0;
+
+ /** Returns true for aggregate functions of type -State.
+ * They are executed as other aggregate functions, but not finalized (return an aggregation state that can be combined with another).
+ */
+ virtual bool is_state() const { return false; }
+
+ /** The inner loop that uses the function pointer is better than using the virtual function.
+ * The reason is that in the case of virtual functions GCC 5.1.2 generates code,
+ * which, at each iteration of the loop, reloads the function address (the offset value in the virtual function table) from memory to the register.
+ * This gives a performance drop on simple queries around 12%.
+ * After the appearance of better compilers, the code can be removed.
+ */
+ using AddFunc = void (*)(const IAggregateFunction*, AggregateDataPtr, const IColumn**, size_t,
+ Arena*);
+ virtual AddFunc get_address_of_add_function() const = 0;
+
+ /** Contains a loop with calls to "add" function. You can collect arguments into array "places"
+ * and do a single call to "add_batch" for devirtualization and inlining.
+ */
+ virtual void add_batch(size_t batch_size, AggregateDataPtr* places, size_t place_offset,
+ const IColumn** columns, Arena* arena) const = 0;
+
+ /** The same for single place.
+ */
+ virtual void add_batch_single_place(size_t batch_size, AggregateDataPtr place,
+ const IColumn** columns, Arena* arena) const = 0;
+
+ // only used at agg reader
+ virtual void add_batch_range(size_t batch_begin, size_t batch_end, AggregateDataPtr place,
+ const IColumn** columns, Arena* arena, bool has_null = false) = 0;
+
+ // only used at window function
+ virtual void add_range_single_place(int64_t partition_start, int64_t partition_end,
+ int64_t frame_start, int64_t frame_end,
+ AggregateDataPtr place, const IColumn** columns,
+ Arena* arena) const = 0;
+
+ /** This is used for runtime code generation to determine, which header files to include in generated source.
+ * Always implement it as
+ * const char * get_header_file_path() const override { return __FILE__; }
+ */
+ virtual const char* get_header_file_path() const = 0;
+
+ const DataTypes& get_argument_types() const { return argument_types; }
+ const Array& get_parameters() const { return parameters; }
+
+protected:
+ DataTypes argument_types;
+ Array parameters;
+};
+
+/// Implement method to obtain an address of 'add' function.
+template <typename Derived>
+class IAggregateFunctionHelper : public IAggregateFunction {
+private:
+ static void add_free(const IAggregateFunction* that, AggregateDataPtr place,
+ const IColumn** columns, size_t row_num, Arena* arena) {
+ static_cast<const Derived&>(*that).add(place, columns, row_num, arena);
+ }
+
+public:
+ IAggregateFunctionHelper(const DataTypes& argument_types_, const Array& parameters_)
+ : IAggregateFunction(argument_types_, parameters_) {}
+
+ AddFunc get_address_of_add_function() const override { return &add_free; }
+
+ void add_batch(size_t batch_size, AggregateDataPtr* places, size_t place_offset,
+ const IColumn** columns, Arena* arena) const override {
+ for (size_t i = 0; i < batch_size; ++i)
+ static_cast<const Derived*>(this)->add(places[i] + place_offset, columns, i, arena);
+ }
+
+ void add_batch_single_place(size_t batch_size, AggregateDataPtr place, const IColumn** columns,
+ Arena* arena) const override {
+ for (size_t i = 0; i < batch_size; ++i)
+ static_cast<const Derived*>(this)->add(place, columns, i, arena);
+ }
+ //now this is use for sum/count/avg/min/max win function, other win function should override this function in class
+ void add_range_single_place(int64_t partition_start, int64_t partition_end, int64_t frame_start,
+ int64_t frame_end, AggregateDataPtr place, const IColumn** columns,
+ Arena* arena) const override {
+ frame_start = std::max<int64_t>(frame_start, partition_start);
+ frame_end = std::min<int64_t>(frame_end, partition_end);
+ for (int64_t i = frame_start; i < frame_end; ++i) {
+ static_cast<const Derived*>(this)->add(place, columns, i, arena);
+ }
+ }
+
+ void add_batch_range(size_t batch_begin, size_t batch_end, AggregateDataPtr place,
+ const IColumn** columns, Arena* arena, bool has_null) override {
+ for (size_t i = batch_begin; i <= batch_end; ++i)
+ static_cast<const Derived*>(this)->add(place, columns, i, arena);
+ }
+};
+
+/// Implements several methods for manipulation with data. T - type of structure with data for aggregation.
+template <typename T, typename Derived>
+class IAggregateFunctionDataHelper : public IAggregateFunctionHelper<Derived> {
+protected:
+ using Data = T;
+
+ static Data& data(AggregateDataPtr __restrict place) { return *reinterpret_cast<Data*>(place); }
+ static const Data& data(ConstAggregateDataPtr __restrict place) {
+ return *reinterpret_cast<const Data*>(place);
+ }
+
+public:
+ IAggregateFunctionDataHelper(const DataTypes& argument_types_, const Array& parameters_)
+ : IAggregateFunctionHelper<Derived>(argument_types_, parameters_) {}
+
+ void create(AggregateDataPtr __restrict place) const override { new (place) Data; }
+
+ void destroy(AggregateDataPtr __restrict place) const noexcept override { data(place).~Data(); }
+
+ bool has_trivial_destructor() const override { return std::is_trivially_destructible_v<Data>; }
+
+ size_t size_of_data() const override { return sizeof(Data); }
+
+ /// NOTE: Currently not used (structures with aggregation state are put without alignment).
+ size_t align_of_data() const override { return alignof(Data); }
+
+ void reset(AggregateDataPtr place) const override {}
+};
+
+using AggregateFunctionPtr = std::shared_ptr<IAggregateFunction>;
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/aggregate_functions/aggregate_function_avg.cpp b/be/src/vec/aggregate_functions/aggregate_function_avg.cpp
new file mode 100644
index 0000000..bb7605b
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_avg.cpp
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/AggregateFunctions/AggregateFunctionAvg.cpp
+// and modified by Doris
+
+#include "vec/aggregate_functions/aggregate_function_avg.h"
+
+#include "common/logging.h"
+#include "vec/aggregate_functions/aggregate_function_simple_factory.h"
+#include "vec/aggregate_functions/factory_helpers.h"
+#include "vec/aggregate_functions/helpers.h"
+
+namespace doris::vectorized {
+
+namespace {
+
+template <typename T>
+struct Avg {
+ using FieldType = std::conditional_t<IsDecimalNumber<T>, Decimal128, NearestFieldType<T>>;
+ using Function = AggregateFunctionAvg<T, AggregateFunctionAvgData<FieldType>>;
+};
+
+template <typename T>
+using AggregateFuncAvg = typename Avg<T>::Function;
+
+AggregateFunctionPtr create_aggregate_function_avg(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable) {
+ assert_no_parameters(name, parameters);
+ assert_unary(name, argument_types);
+
+ AggregateFunctionPtr res;
+ DataTypePtr data_type = argument_types[0];
+ if (is_decimal(data_type))
+ res.reset(
+ create_with_decimal_type<AggregateFuncAvg>(*data_type, *data_type, argument_types));
+ else
+ res.reset(create_with_numeric_type<AggregateFuncAvg>(*data_type, argument_types));
+
+ if (!res) {
+ LOG(WARNING) << fmt::format("Illegal type {} of argument for aggregate function {}",
+ argument_types[0]->get_name(), name);
+ }
+ return res;
+}
+
+} // namespace
+
+void register_aggregate_function_avg(AggregateFunctionSimpleFactory& factory) {
+ factory.register_function("avg", create_aggregate_function_avg);
+}
+} // namespace doris::vectorized
diff --git a/be/src/vec/aggregate_functions/aggregate_function_avg.h b/be/src/vec/aggregate_functions/aggregate_function_avg.h
new file mode 100644
index 0000000..18584ee
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_avg.h
@@ -0,0 +1,128 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/AggregateFunctions/AggregateFunctionAvg.h
+// and modified by Doris
+
+#pragma once
+
+#include "common/status.h"
+#include "vec/aggregate_functions/aggregate_function.h"
+#include "vec/columns/columns_number.h"
+#include "vec/data_types/data_type_decimal.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/io/io_helper.h"
+
+namespace doris::vectorized {
+
+template <typename T>
+struct AggregateFunctionAvgData {
+ T sum = 0;
+ UInt64 count = 0;
+
+ template <typename ResultT>
+ ResultT result() const {
+ if constexpr (std::is_floating_point_v<ResultT>)
+ if constexpr (std::numeric_limits<ResultT>::is_iec559)
+ return static_cast<ResultT>(sum) / count; /// allow division by zero
+
+ if (!count)
+ throw Exception("AggregateFunctionAvg with zero values", TStatusCode::VEC_LOGIC_ERROR);
+ return static_cast<ResultT>(sum) / count;
+ }
+
+ void write(BufferWritable& buf) const {
+ write_binary(sum, buf);
+ write_binary(count, buf);
+ }
+
+ void read(BufferReadable& buf) {
+ read_binary(sum, buf);
+ read_binary(count, buf);
+ }
+};
+
+/// Calculates arithmetic mean of numbers.
+template <typename T, typename Data>
+class AggregateFunctionAvg final
+ : public IAggregateFunctionDataHelper<Data, AggregateFunctionAvg<T, Data>> {
+public:
+ using ResultType = std::conditional_t<IsDecimalNumber<T>, Decimal128, Float64>;
+ using ResultDataType = std::conditional_t<IsDecimalNumber<T>, DataTypeDecimal<Decimal128>,
+ DataTypeNumber<Float64>>;
+ using ColVecType = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<T>, ColumnVector<T>>;
+ using ColVecResult = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<Decimal128>,
+ ColumnVector<Float64>>;
+
+ /// ctor for native types
+ AggregateFunctionAvg(const DataTypes& argument_types_)
+ : IAggregateFunctionDataHelper<Data, AggregateFunctionAvg<T, Data>>(argument_types_,
+ {}),
+ scale(0) {}
+
+ /// ctor for Decimals
+ AggregateFunctionAvg(const IDataType& data_type, const DataTypes& argument_types_)
+ : IAggregateFunctionDataHelper<Data, AggregateFunctionAvg<T, Data>>(argument_types_,
+ {}),
+ scale(get_decimal_scale(data_type)) {}
+
+ String get_name() const override { return "avg"; }
+
+ DataTypePtr get_return_type() const override {
+ if constexpr (IsDecimalNumber<T>)
+ return std::make_shared<ResultDataType>(ResultDataType::max_precision(), scale);
+ else
+ return std::make_shared<ResultDataType>();
+ }
+
+ void add(AggregateDataPtr __restrict place, const IColumn** columns, size_t row_num,
+ Arena*) const override {
+ const auto& column = static_cast<const ColVecType&>(*columns[0]);
+ this->data(place).sum += column.get_data()[row_num];
+ ++this->data(place).count;
+ }
+
+ void reset(AggregateDataPtr place) const override {
+ this->data(place).sum = 0;
+ this->data(place).count = 0;
+ }
+
+ void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena*) const override {
+ this->data(place).sum += this->data(rhs).sum;
+ this->data(place).count += this->data(rhs).count;
+ }
+
+ void serialize(ConstAggregateDataPtr __restrict place, BufferWritable& buf) const override {
+ this->data(place).write(buf);
+ }
+
+ void deserialize(AggregateDataPtr __restrict place, BufferReadable& buf, Arena*) const override {
+ this->data(place).read(buf);
+ }
+
+ void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override {
+ auto& column = static_cast<ColVecResult&>(to);
+ column.get_data().push_back(this->data(place).template result<ResultType>());
+ }
+
+ const char* get_header_file_path() const override { return __FILE__; }
+
+private:
+ UInt32 scale;
+};
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/aggregate_functions/aggregate_function_bitmap.cpp b/be/src/vec/aggregate_functions/aggregate_function_bitmap.cpp
new file mode 100644
index 0000000..d110b09
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_bitmap.cpp
@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/aggregate_functions/aggregate_function_bitmap.h"
+
+#include "vec/aggregate_functions/aggregate_function_simple_factory.h"
+
+namespace doris::vectorized {
+
+template <bool nullable, template <bool, typename> class AggregateFunctionTemplate>
+static IAggregateFunction* createWithIntDataType(const DataTypes& argument_type) {
+ auto type = argument_type[0].get();
+ if (type->is_nullable()) {
+ type = assert_cast<const DataTypeNullable*>(type)->get_nested_type().get();
+ }
+ WhichDataType which(type);
+ if (which.idx == TypeIndex::Int8)
+ return new AggregateFunctionTemplate<nullable, ColumnVector<Int8>>(argument_type);
+ if (which.idx == TypeIndex::Int16)
+ return new AggregateFunctionTemplate<nullable, ColumnVector<Int16>>(argument_type);
+ if (which.idx == TypeIndex::Int32)
+ return new AggregateFunctionTemplate<nullable, ColumnVector<Int32>>(argument_type);
+ if (which.idx == TypeIndex::Int64)
+ return new AggregateFunctionTemplate<nullable, ColumnVector<Int64>>(argument_type);
+ return nullptr;
+}
+
+AggregateFunctionPtr create_aggregate_function_bitmap_union(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable) {
+ return std::make_shared<AggregateFunctionBitmapOp<AggregateFunctionBitmapUnionOp>>(
+ argument_types);
+}
+
+AggregateFunctionPtr create_aggregate_function_bitmap_intersect(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable) {
+ return std::make_shared<AggregateFunctionBitmapOp<AggregateFunctionBitmapIntersectOp>>(
+ argument_types);
+}
+template <bool nullable>
+AggregateFunctionPtr create_aggregate_function_bitmap_union_count(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable) {
+ return std::make_shared<AggregateFunctionBitmapCount<nullable, ColumnBitmap>>(argument_types);
+}
+
+template <bool nullable>
+AggregateFunctionPtr create_aggregate_function_bitmap_union_int(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable) {
+ return std::shared_ptr<IAggregateFunction>(
+ createWithIntDataType<nullable, AggregateFunctionBitmapCount>(argument_types));
+}
+
+void register_aggregate_function_bitmap(AggregateFunctionSimpleFactory& factory) {
+ factory.register_function("bitmap_union", create_aggregate_function_bitmap_union);
+ factory.register_function("bitmap_intersect", create_aggregate_function_bitmap_intersect);
+ factory.register_function("bitmap_union_count",
+ create_aggregate_function_bitmap_union_count<false>);
+ factory.register_function("bitmap_union_count",
+ create_aggregate_function_bitmap_union_count<true>, true);
+
+ factory.register_function("bitmap_union_int",
+ create_aggregate_function_bitmap_union_int<false>);
+ factory.register_function("bitmap_union_int", create_aggregate_function_bitmap_union_int<true>,
+ true);
+}
+} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/src/vec/aggregate_functions/aggregate_function_bitmap.h b/be/src/vec/aggregate_functions/aggregate_function_bitmap.h
new file mode 100644
index 0000000..a2e43e5
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_bitmap.h
@@ -0,0 +1,175 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+#include <istream>
+#include <ostream>
+
+#include "vec/aggregate_functions/aggregate_function.h"
+#include "vec/columns/column_complex.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/common/assert_cast.h"
+#include "vec/data_types/data_type_bitmap.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/io/io_helper.h"
+
+namespace doris::vectorized {
+
+struct AggregateFunctionBitmapUnionOp {
+ static constexpr auto name = "bitmap_union";
+
+ template <typename T>
+ static void add(BitmapValue& res, const T& data) {
+ res.add(data);
+ }
+
+ static void add(BitmapValue& res, const BitmapValue& data) { res |= data; }
+
+ static void merge(BitmapValue& res, const BitmapValue& data) { res |= data; }
+};
+
+struct AggregateFunctionBitmapIntersectOp {
+ static constexpr auto name = "bitmap_intersect";
+ static void add(BitmapValue& res, const BitmapValue& data) { res &= data; }
+
+ static void merge(BitmapValue& res, const BitmapValue& data) { res &= data; }
+};
+
+template <typename Op>
+struct AggregateFunctionBitmapData {
+ BitmapValue value;
+
+ template <typename T>
+ void add(const T& data) {
+ Op::add(value, data);
+ }
+
+ void merge(const BitmapValue& data) { Op::merge(value, data); }
+
+ void write(BufferWritable& buf) const { DataTypeBitMap::serialize_as_stream(value, buf); }
+
+ void read(BufferReadable& buf) { DataTypeBitMap::deserialize_as_stream(value, buf); }
+
+ BitmapValue& get() { return value; }
+};
+
+template <typename Op>
+class AggregateFunctionBitmapOp final
+ : public IAggregateFunctionDataHelper<AggregateFunctionBitmapData<Op>,
+ AggregateFunctionBitmapOp<Op>> {
+public:
+ using ResultDataType = BitmapValue;
+ using ColVecType = ColumnBitmap;
+ using ColVecResult = ColumnBitmap;
+
+ String get_name() const override { return Op::name; }
+
+ AggregateFunctionBitmapOp(const DataTypes& argument_types_)
+ : IAggregateFunctionDataHelper<AggregateFunctionBitmapData<Op>,
+ AggregateFunctionBitmapOp<Op>>(argument_types_, {}) {}
+
+ DataTypePtr get_return_type() const override { return std::make_shared<DataTypeBitMap>(); }
+
+ void add(AggregateDataPtr __restrict place, const IColumn** columns, size_t row_num,
+ Arena*) const override {
+ const auto& column = static_cast<const ColVecType&>(*columns[0]);
+ this->data(place).add(column.get_data()[row_num]);
+ }
+
+ void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena*) const override {
+ this->data(place).merge(
+ const_cast<AggregateFunctionBitmapData<Op>&>(this->data(rhs)).get());
+ }
+
+ void serialize(ConstAggregateDataPtr __restrict place, BufferWritable& buf) const override {
+ this->data(place).write(buf);
+ }
+
+ void deserialize(AggregateDataPtr __restrict place, BufferReadable& buf, Arena*) const override {
+ this->data(place).read(buf);
+ }
+
+ void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override {
+ auto& column = static_cast<ColVecResult&>(to);
+ column.get_data().push_back(
+ const_cast<AggregateFunctionBitmapData<Op>&>(this->data(place)).get());
+ }
+
+ const char* get_header_file_path() const override { return __FILE__; }
+};
+
+template <bool nullable, typename ColVecType>
+class AggregateFunctionBitmapCount final
+ : public IAggregateFunctionDataHelper<
+ AggregateFunctionBitmapData<AggregateFunctionBitmapUnionOp>,
+ AggregateFunctionBitmapCount<nullable, ColVecType>> {
+public:
+ // using ColVecType = ColumnBitmap;
+ using ColVecResult = ColumnVector<Int64>;
+ using AggFunctionData = AggregateFunctionBitmapData<AggregateFunctionBitmapUnionOp>;
+
+ AggregateFunctionBitmapCount(const DataTypes& argument_types_)
+ : IAggregateFunctionDataHelper<
+ AggregateFunctionBitmapData<AggregateFunctionBitmapUnionOp>,
+ AggregateFunctionBitmapCount<nullable, ColVecType>>(argument_types_, {}) {}
+
+ String get_name() const override { return "count"; }
+ DataTypePtr get_return_type() const override { return std::make_shared<DataTypeInt64>(); }
+
+ void add(AggregateDataPtr __restrict place, const IColumn** columns, size_t row_num,
+ Arena*) const override {
+ if constexpr (nullable) {
+ auto& nullable_column = assert_cast<const ColumnNullable&>(*columns[0]);
+ if (!nullable_column.is_null_at(row_num)) {
+ const auto& column =
+ static_cast<const ColVecType&>(nullable_column.get_nested_column());
+ this->data(place).add(column.get_data()[row_num]);
+ }
+ } else {
+ const auto& column = static_cast<const ColVecType&>(*columns[0]);
+ this->data(place).add(column.get_data()[row_num]);
+ }
+ }
+
+ void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena*) const override {
+ this->data(place).merge(const_cast<AggFunctionData&>(this->data(rhs)).get());
+ }
+
+ void serialize(ConstAggregateDataPtr __restrict place, BufferWritable& buf) const override {
+ this->data(place).write(buf);
+ }
+
+ void deserialize(AggregateDataPtr __restrict place, BufferReadable& buf, Arena*) const override {
+ this->data(place).read(buf);
+ }
+
+ void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override {
+ auto& value_data = const_cast<AggFunctionData&>(this->data(place)).get();
+ auto& column = static_cast<ColVecResult&>(to);
+ column.get_data().push_back(value_data.cardinality());
+ }
+
+ const char* get_header_file_path() const override { return __FILE__; }
+};
+
+AggregateFunctionPtr create_aggregate_function_bitmap_union(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable);
+
+} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/src/vec/aggregate_functions/aggregate_function_combinator.h b/be/src/vec/aggregate_functions/aggregate_function_combinator.h
new file mode 100644
index 0000000..77d465b
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_combinator.h
@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/AggregateFunctions/IAggregateFunctionCombinator.h
+// and modified by Doris
+
+#pragma once
+
+#include <vec/aggregate_functions/aggregate_function.h>
+#include <vec/data_types/data_type.h>
+
+#include <memory>
+
+namespace doris::vectorized {
+
+/** Aggregate function combinator allows to take one aggregate function
+ * and transform it to another aggregate function.
+ *
+ * In SQL language they are used as suffixes for existing aggregate functions.
+ *
+ * Example: -If combinator takes an aggregate function and transforms it
+ * to aggregate function with additional argument at end (condition),
+ * that will pass values to original aggregate function when the condition is true.
+ *
+ * More examples:
+ *
+ * sum(x) - calculate sum of x
+ * sumIf(x, cond) - calculate sum of x for rows where condition is true.
+ * sumArray(arr) - calculate sum of all elements of arrays.
+ *
+ * PS. Please don't mess it with so called "combiner" - totally unrelated notion from Hadoop world.
+ * "combining" - merging the states of aggregate functions - is supported naturally in ClickHouse.
+ */
+
+class IAggregateFunctionCombinator {
+public:
+ virtual String get_name() const = 0;
+
+ virtual bool is_for_internal_usage_only() const { return false; }
+
+ /** From the arguments for combined function (ex: UInt64, UInt8 for sumIf),
+ * get the arguments for nested function (ex: UInt64 for sum).
+ * If arguments are not suitable for combined function, throw an exception.
+ */
+ virtual DataTypes transform_arguments(const DataTypes& arguments) const { return arguments; }
+
+ /** From the parameters for combined function,
+ * get the parameters for nested function.
+ * If arguments are not suitable for combined function, throw an exception.
+ */
+ virtual Array transform_parameters(const Array& parameters) const { return parameters; }
+
+ /** Create combined aggregate function (ex: sumIf)
+ * from nested function (ex: sum)
+ * and arguments for combined agggregate function (ex: UInt64, UInt8 for sumIf).
+ * It's assumed that function transform_arguments was called before this function and 'arguments' are validated.
+ */
+ virtual AggregateFunctionPtr transform_aggregate_function(
+ const AggregateFunctionPtr& nested_function, const DataTypes& arguments,
+ const Array& params, const bool result_is_nullable) const = 0;
+
+ virtual ~IAggregateFunctionCombinator() {}
+};
+
+using AggregateFunctionCombinatorPtr = std::shared_ptr<const IAggregateFunctionCombinator>;
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/aggregate_functions/aggregate_function_count.cpp b/be/src/vec/aggregate_functions/aggregate_function_count.cpp
new file mode 100644
index 0000000..aa5d006
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_count.cpp
@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/AggregateFunctions/AggregateFunctionCount.cpp
+// and modified by Doris
+
+#include "vec/aggregate_functions/aggregate_function_count.h"
+
+#include "vec/aggregate_functions/aggregate_function_simple_factory.h"
+#include "vec/aggregate_functions/factory_helpers.h"
+
+namespace doris::vectorized {
+
+AggregateFunctionPtr create_aggregate_function_count(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable) {
+ assert_no_parameters(name, parameters);
+ assert_arity_at_most<1>(name, argument_types);
+
+ return std::make_shared<AggregateFunctionCount>(argument_types);
+}
+
+AggregateFunctionPtr create_aggregate_function_count_not_null_unary(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable) {
+ assert_arity_at_most<1>(name, argument_types);
+
+ return std::make_shared<AggregateFunctionCountNotNullUnary>(argument_types);
+}
+
+void register_aggregate_function_count(AggregateFunctionSimpleFactory& factory) {
+ factory.register_function("count", create_aggregate_function_count);
+ factory.register_function("count", create_aggregate_function_count_not_null_unary, true);
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/aggregate_functions/aggregate_function_count.h b/be/src/vec/aggregate_functions/aggregate_function_count.h
new file mode 100644
index 0000000..fd096dc
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_count.h
@@ -0,0 +1,122 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/AggregateFunctions/AggregateFunctionCount.h
+// and modified by Doris
+
+#pragma once
+
+#include <array>
+
+#include "common/logging.h"
+#include "vec/aggregate_functions/aggregate_function.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/common/assert_cast.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/io/io_helper.h"
+
+namespace doris::vectorized {
+
+struct AggregateFunctionCountData {
+ UInt64 count = 0;
+};
+
+/// Simply count number of calls.
+class AggregateFunctionCount final
+ : public IAggregateFunctionDataHelper<AggregateFunctionCountData, AggregateFunctionCount> {
+public:
+ AggregateFunctionCount(const DataTypes& argument_types_)
+ : IAggregateFunctionDataHelper(argument_types_, {}) {}
+
+ String get_name() const override { return "count"; }
+
+ DataTypePtr get_return_type() const override { return std::make_shared<DataTypeInt64>(); }
+
+ void add(AggregateDataPtr __restrict place, const IColumn**, size_t, Arena*) const override {
+ ++data(place).count;
+ }
+
+ void reset(AggregateDataPtr place) const override {
+ this->data(place).count = 0;
+ }
+
+ void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena*) const override {
+ data(place).count += data(rhs).count;
+ }
+
+ void serialize(ConstAggregateDataPtr __restrict place, BufferWritable& buf) const override {
+ write_var_uint(data(place).count, buf);
+ }
+
+ void deserialize(AggregateDataPtr __restrict place, BufferReadable& buf, Arena*) const override {
+ read_var_uint(data(place).count, buf);
+ }
+
+ void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override {
+ assert_cast<ColumnInt64&>(to).get_data().push_back(data(place).count);
+ }
+
+ const char* get_header_file_path() const override { return __FILE__; }
+};
+
+/// Simply count number of not-NULL values.
+class AggregateFunctionCountNotNullUnary final
+ : public IAggregateFunctionDataHelper<AggregateFunctionCountData,
+ AggregateFunctionCountNotNullUnary> {
+public:
+ AggregateFunctionCountNotNullUnary(const DataTypes& argument_types_)
+ : IAggregateFunctionDataHelper(argument_types_, {}) {}
+
+ String get_name() const override { return "count"; }
+
+ DataTypePtr get_return_type() const override { return std::make_shared<DataTypeInt64>(); }
+
+ void add(AggregateDataPtr __restrict place, const IColumn** columns, size_t row_num,
+ Arena*) const override {
+ data(place).count += !assert_cast<const ColumnNullable&>(*columns[0]).is_null_at(row_num);
+ }
+
+ void reset(AggregateDataPtr place) const override {
+ data(place).count = 0;
+ }
+
+ void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena*) const override {
+ data(place).count += data(rhs).count;
+ }
+
+ void serialize(ConstAggregateDataPtr __restrict place, BufferWritable& buf) const override {
+ write_var_uint(data(place).count, buf);
+ }
+
+ void deserialize(AggregateDataPtr __restrict place, BufferReadable& buf, Arena*) const override {
+ read_var_uint(data(place).count, buf);
+ }
+
+ void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override {
+ if (to.is_nullable()) {
+ auto& null_column = assert_cast<ColumnNullable &>(to);
+ null_column.get_null_map_data().push_back(0);
+ assert_cast<ColumnInt64 &>(null_column.get_nested_column()).get_data().push_back(data(place).count);
+ } else {
+ assert_cast<ColumnInt64 &>(to).get_data().push_back(data(place).count);
+ }
+ }
+
+ const char* get_header_file_path() const override { return __FILE__; }
+};
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/aggregate_functions/aggregate_function_distinct.cpp b/be/src/vec/aggregate_functions/aggregate_function_distinct.cpp
new file mode 100644
index 0000000..af64a3a
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_distinct.cpp
@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/AggregateFunctions/AggregateFunctionDistinct.cpp
+// and modified by Doris
+
+#include <algorithm>
+#include <boost/algorithm/string.hpp>
+
+#include "vec/aggregate_functions/aggregate_function_combinator.h"
+#include "vec/aggregate_functions/aggregate_function_distinct.h"
+#include "vec/aggregate_functions/aggregate_function_simple_factory.h"
+#include "vec/aggregate_functions/helpers.h"
+#include "vec/common/typeid_cast.h"
+#include "vec/data_types/data_type_nullable.h"
+
+namespace doris::vectorized {
+
+class AggregateFunctionCombinatorDistinct final : public IAggregateFunctionCombinator {
+public:
+ String get_name() const override { return "Distinct"; }
+
+ DataTypes transform_arguments(const DataTypes& arguments) const override {
+ if (arguments.empty()) {
+ LOG(FATAL)
+ << "Incorrect number of arguments for aggregate function with Distinct suffix";
+ }
+ return arguments;
+ }
+
+ AggregateFunctionPtr transform_aggregate_function(const AggregateFunctionPtr& nested_function,
+ const DataTypes& arguments,
+ const Array& params,
+ const bool result_is_nullable) const override {
+ DCHECK(nested_function != nullptr);
+ if (nested_function == nullptr) return nullptr;
+
+ AggregateFunctionPtr res;
+ if (arguments.size() == 1) {
+ res.reset(create_with_numeric_type<AggregateFunctionDistinct,
+ AggregateFunctionDistinctSingleNumericData>(
+ *arguments[0], nested_function, arguments));
+
+ if (res) return res;
+
+ if (arguments[0]->is_value_unambiguously_represented_in_contiguous_memory_region())
+ return std::make_shared<AggregateFunctionDistinct<
+ AggregateFunctionDistinctSingleGenericData<true>>>(nested_function,
+ arguments);
+ else
+ return std::make_shared<AggregateFunctionDistinct<
+ AggregateFunctionDistinctSingleGenericData<false>>>(nested_function,
+ arguments);
+ }
+
+ return std::make_shared<
+ AggregateFunctionDistinct<AggregateFunctionDistinctMultipleGenericData>>(
+ nested_function, arguments);
+ }
+};
+
+const std::string DISTINCT_FUNCTION_PREFIX = "multi_distinct_";
+
+void register_aggregate_function_combinator_distinct(AggregateFunctionSimpleFactory& factory) {
+ AggregateFunctionCreator creator = [&](const std::string& name, const DataTypes& types,
+ const Array& params, const bool result_is_nullable) {
+ // 1. we should get not nullable types;
+ DataTypes nested_types(types.size());
+ std::transform(types.begin(), types.end(), nested_types.begin(),
+ [](const auto& e) { return remove_nullable(e); });
+ auto function_combinator = std::make_shared<AggregateFunctionCombinatorDistinct>();
+ auto transform_arguments = function_combinator->transform_arguments(nested_types);
+ if (!boost::algorithm::starts_with(name, DISTINCT_FUNCTION_PREFIX)) {
+ return AggregateFunctionPtr();
+ }
+ auto nested_function_name = name.substr(DISTINCT_FUNCTION_PREFIX.size());
+ auto nested_function = factory.get(nested_function_name, transform_arguments, params);
+ return function_combinator->transform_aggregate_function(nested_function, types, params, result_is_nullable);
+ };
+ factory.register_distinct_function_combinator(creator, DISTINCT_FUNCTION_PREFIX);
+}
+} // namespace doris::vectorized
diff --git a/be/src/vec/aggregate_functions/aggregate_function_distinct.h b/be/src/vec/aggregate_functions/aggregate_function_distinct.h
new file mode 100644
index 0000000..6502b48
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_distinct.h
@@ -0,0 +1,224 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/AggregateFunctions/AggregateFunctionDistinct.h
+// and modified by Doris
+
+#pragma once
+
+#include "vec/aggregate_functions/aggregate_function.h"
+#include "vec/aggregate_functions/key_holder_helpers.h"
+#include "vec/common/aggregation_common.h"
+#include "vec/common/assert_cast.h"
+#include "vec/common/field_visitors.h"
+#include "vec/common/hash_table/hash_set.h"
+#include "vec/common/hash_table/hash_table.h"
+#include "vec/common/sip_hash.h"
+#include "vec/io/io_helper.h"
+
+namespace doris::vectorized {
+
+template <typename T>
+struct AggregateFunctionDistinctSingleNumericData {
+ /// When creating, the hash table must be small.
+ using Set = HashSetWithStackMemory<T, DefaultHash<T>, 4>;
+ using Self = AggregateFunctionDistinctSingleNumericData<T>;
+ Set set;
+
+ void add(const IColumn** columns, size_t /* columns_num */, size_t row_num, Arena*) {
+ const auto& vec = assert_cast<const ColumnVector<T>&>(*columns[0]).get_data();
+ set.insert(vec[row_num]);
+ }
+
+ void merge(const Self& rhs, Arena*) { set.merge(rhs.set); }
+
+ void serialize(BufferWritable& buf) const { set.write(buf); }
+
+ void deserialize(BufferReadable& buf, Arena*) { set.read(buf); }
+
+ MutableColumns get_arguments(const DataTypes& argument_types) const {
+ MutableColumns argument_columns;
+ argument_columns.emplace_back(argument_types[0]->create_column());
+ for (const auto& elem : set) argument_columns[0]->insert(elem.get_value());
+
+ return argument_columns;
+ }
+};
+
+struct AggregateFunctionDistinctGenericData {
+ /// When creating, the hash table must be small.
+ using Set = HashSetWithSavedHashWithStackMemory<StringRef, StringRefHash, 4>;
+ using Self = AggregateFunctionDistinctGenericData;
+ Set set;
+
+ void merge(const Self& rhs, Arena* arena) {
+ Set::LookupResult it;
+ bool inserted;
+ for (const auto& elem : rhs.set)
+ set.emplace(ArenaKeyHolder{elem.get_value(), *arena}, it, inserted);
+ }
+
+ void serialize(BufferWritable& buf) const {
+ write_var_uint(set.size(), buf);
+ for (const auto& elem : set)
+ write_string_binary(elem.get_value(), buf);
+ }
+
+ void deserialize(BufferReadable& buf, Arena* arena) {
+ size_t size;
+ read_var_uint(size, buf);
+
+ StringRef ref;
+ for (size_t i = 0; i < size; ++i) {
+ read_string_binary(ref, buf);
+ set.insert(ref);
+ }
+ }
+};
+
+template <bool is_plain_column>
+struct AggregateFunctionDistinctSingleGenericData : public AggregateFunctionDistinctGenericData {
+ void add(const IColumn** columns, size_t /* columns_num */, size_t row_num, Arena* arena) {
+ Set::LookupResult it;
+ bool inserted;
+ auto key_holder = get_key_holder<is_plain_column>(*columns[0], row_num, *arena);
+ set.emplace(key_holder, it, inserted);
+ }
+
+ MutableColumns get_arguments(const DataTypes& argument_types) const {
+ MutableColumns argument_columns;
+ argument_columns.emplace_back(argument_types[0]->create_column());
+ for (const auto& elem : set)
+ deserialize_and_insert<is_plain_column>(elem.get_value(), *argument_columns[0]);
+
+ return argument_columns;
+ }
+};
+
+struct AggregateFunctionDistinctMultipleGenericData : public AggregateFunctionDistinctGenericData {
+ void add(const IColumn** columns, size_t columns_num, size_t row_num, Arena* arena) {
+ const char* begin = nullptr;
+ StringRef value(begin, 0);
+ for (size_t i = 0; i < columns_num; ++i) {
+ auto cur_ref = columns[i]->serialize_value_into_arena(row_num, *arena, begin);
+ value.data = cur_ref.data - value.size;
+ value.size += cur_ref.size;
+ }
+
+ Set::LookupResult it;
+ bool inserted;
+ auto key_holder = SerializedKeyHolder{value, *arena};
+ set.emplace(key_holder, it, inserted);
+ }
+
+ MutableColumns get_arguments(const DataTypes& argument_types) const {
+ MutableColumns argument_columns(argument_types.size());
+ for (size_t i = 0; i < argument_types.size(); ++i)
+ argument_columns[i] = argument_types[i]->create_column();
+
+ for (const auto& elem : set) {
+ const char* begin = elem.get_value().data;
+ for (auto& column : argument_columns)
+ begin = column->deserialize_and_insert_from_arena(begin);
+ }
+
+ return argument_columns;
+ }
+};
+
+/** Adaptor for aggregate functions.
+ * Adding -Distinct suffix to aggregate function
+**/
+template <typename Data>
+class AggregateFunctionDistinct
+ : public IAggregateFunctionDataHelper<Data, AggregateFunctionDistinct<Data>> {
+private:
+ static constexpr auto prefix_size = sizeof(Data);
+ AggregateFunctionPtr nested_func;
+ size_t arguments_num;
+
+ AggregateDataPtr get_nested_place(AggregateDataPtr __restrict place) const noexcept {
+ return place + prefix_size;
+ }
+
+ ConstAggregateDataPtr get_nested_place(ConstAggregateDataPtr __restrict place) const noexcept {
+ return place + prefix_size;
+ }
+
+public:
+ AggregateFunctionDistinct(AggregateFunctionPtr nested_func_, const DataTypes& arguments)
+ : IAggregateFunctionDataHelper<Data, AggregateFunctionDistinct>(
+ arguments, nested_func_->get_parameters()),
+ nested_func(nested_func_),
+ arguments_num(arguments.size()) {}
+
+ void add(AggregateDataPtr __restrict place, const IColumn** columns, size_t row_num,
+ Arena* arena) const override {
+ this->data(place).add(columns, arguments_num, row_num, arena);
+ }
+
+ void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs,
+ Arena* arena) const override {
+ this->data(place).merge(this->data(rhs), arena);
+ }
+
+ void serialize(ConstAggregateDataPtr __restrict place, BufferWritable& buf) const override {
+ this->data(place).serialize(buf);
+ }
+
+ void deserialize(AggregateDataPtr __restrict place, BufferReadable& buf, Arena* arena) const override {
+ this->data(place).deserialize(buf, arena);
+ }
+
+ // void insert_result_into(AggregateDataPtr place, IColumn & to, Arena * arena) const override
+ void insert_result_into(ConstAggregateDataPtr targetplace, IColumn& to) const override {
+ auto place = const_cast<AggregateDataPtr>(targetplace);
+ auto arguments = this->data(place).get_arguments(this->argument_types);
+ ColumnRawPtrs arguments_raw(arguments.size());
+ for (size_t i = 0; i < arguments.size(); ++i) arguments_raw[i] = arguments[i].get();
+
+ assert(!arguments.empty());
+ // nested_func->add_batch_single_place(arguments[0]->size(), get_nested_place(place), arguments_raw.data(), arena);
+ // nested_func->insert_result_into(get_nested_place(place), to, arena);
+
+ nested_func->add_batch_single_place(arguments[0]->size(), get_nested_place(place),
+ arguments_raw.data(), nullptr);
+ nested_func->insert_result_into(get_nested_place(place), to);
+ }
+
+ size_t size_of_data() const override { return prefix_size + nested_func->size_of_data(); }
+
+ void create(AggregateDataPtr __restrict place) const override {
+ new (place) Data;
+ nested_func->create(get_nested_place(place));
+ }
+
+ void destroy(AggregateDataPtr __restrict place) const noexcept override {
+ this->data(place).~Data();
+ nested_func->destroy(get_nested_place(place));
+ }
+
+ String get_name() const override { return nested_func->get_name() + "Distinct"; }
+
+ DataTypePtr get_return_type() const override { return nested_func->get_return_type(); }
+
+ bool allocates_memory_in_arena() const override { return true; }
+
+ const char* get_header_file_path() const override { return __FILE__; }
+};
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.cpp b/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.cpp
new file mode 100644
index 0000000..3b2aba0
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.cpp
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/aggregate_functions/aggregate_function_hll_union_agg.h"
+
+#include "vec/aggregate_functions/aggregate_function_simple_factory.h"
+#include "vec/aggregate_functions/factory_helpers.h"
+
+namespace doris::vectorized {
+
+AggregateFunctionPtr create_aggregate_function_HLL_union_agg(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable) {
+ assert_no_parameters(name, parameters);
+ assert_arity_at_most<1>(name, argument_types);
+
+ return std::make_shared<AggregateFunctionHLLUnionAgg>(argument_types);
+}
+
+AggregateFunctionPtr create_aggregate_function_HLL_union(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable) {
+ assert_no_parameters(name, parameters);
+ assert_arity_at_most<1>(name, argument_types);
+
+ return std::make_shared<AggregateFunctionHLLUnion>(argument_types);
+}
+
+void register_aggregate_function_HLL_union_agg(AggregateFunctionSimpleFactory& factory) {
+ factory.register_function("hll_union_agg", create_aggregate_function_HLL_union_agg);
+ factory.register_function("hll_union", create_aggregate_function_HLL_union);
+ factory.register_function("hll_raw_agg", create_aggregate_function_HLL_union);
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.h b/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.h
new file mode 100644
index 0000000..f71a1f5
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.h
@@ -0,0 +1,133 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <istream>
+#include <ostream>
+#include <type_traits>
+
+#include "exprs/hll_function.h"
+#include "olap/hll.h"
+#include "util/slice.h"
+#include "vec/aggregate_functions/aggregate_function.h"
+#include "vec/columns/column_string.h"
+#include "vec/columns/column_vector.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/io/io_helper.h"
+
+namespace doris::vectorized {
+
+struct AggregateFunctionHLLData {
+ HyperLogLog dst_hll {};
+
+ void add(const StringRef& src) { dst_hll.merge(HyperLogLog(Slice(src.data, src.size))); }
+
+ void merge(const AggregateFunctionHLLData& rhs) { dst_hll.merge(rhs.dst_hll); }
+
+ void write(BufferWritable& buf) const {
+ std::string result(dst_hll.max_serialized_size(), '0');
+ int size = dst_hll.serialize((uint8_t*)result.c_str());
+ result.resize(size);
+ write_binary(result, buf);
+ }
+
+ void read(BufferReadable& buf) {
+ StringRef ref;
+ read_binary(ref, buf);
+ dst_hll.deserialize(Slice(ref.data, ref.size));
+ }
+
+ Int64 get_cardinality() const { return dst_hll.estimate_cardinality(); }
+
+ std::string get() const {
+ std::string result(dst_hll.max_serialized_size(), '0');
+ int size = dst_hll.serialize((uint8_t*)result.c_str());
+ result.resize(size);
+
+ return result;
+ }
+};
+
+class AggregateFunctionHLLUnionAgg
+ : public IAggregateFunctionDataHelper<AggregateFunctionHLLData,
+ AggregateFunctionHLLUnionAgg> {
+public:
+ virtual String get_name() const override { return "hll_union_agg"; }
+
+ AggregateFunctionHLLUnionAgg(const DataTypes& argument_types_)
+ : IAggregateFunctionDataHelper(argument_types_, {}) {}
+
+ AggregateFunctionHLLUnionAgg(const IDataType& data_type, const DataTypes& argument_types_)
+ : IAggregateFunctionDataHelper(argument_types_, {}) {}
+
+ virtual DataTypePtr get_return_type() const override {
+ return std::make_shared<DataTypeInt64>();
+ }
+
+ void add(AggregateDataPtr __restrict place, const IColumn** columns, size_t row_num,
+ Arena*) const override {
+ const auto& column = static_cast<const ColumnString&>(*columns[0]);
+ this->data(place).add(column.get_data_at(row_num));
+ }
+
+ void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena*) const override {
+ this->data(place).merge(this->data(rhs));
+ }
+
+ void serialize(ConstAggregateDataPtr __restrict place, BufferWritable& buf) const override {
+ this->data(place).write(buf);
+ }
+
+ void deserialize(AggregateDataPtr __restrict place, BufferReadable& buf, Arena*) const override {
+ this->data(place).read(buf);
+ }
+
+ virtual void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override {
+ auto& column = static_cast<ColumnVector<Int64>&>(to);
+ column.get_data().push_back(this->data(place).get_cardinality());
+ }
+
+ const char* get_header_file_path() const override { return __FILE__; }
+};
+
+class AggregateFunctionHLLUnion final : public AggregateFunctionHLLUnionAgg {
+public:
+ String get_name() const override { return "hll_union"; }
+
+ AggregateFunctionHLLUnion(const DataTypes& argument_types_)
+ : AggregateFunctionHLLUnionAgg {argument_types_} {}
+
+ AggregateFunctionHLLUnion(const IDataType& data_type, const DataTypes& argument_types_)
+ : AggregateFunctionHLLUnionAgg(data_type, argument_types_) {}
+
+ DataTypePtr get_return_type() const override { return std::make_shared<DataTypeString>(); }
+
+ void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override {
+ auto& column = static_cast<ColumnString&>(to);
+ auto result = this->data(place).get();
+ column.insert_data(result.c_str(), result.length());
+ }
+};
+
+AggregateFunctionPtr create_aggregate_function_HLL_union(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable);
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp b/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp
new file mode 100644
index 0000000..813674a
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp
@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/AggregateFunctions/AggregateFunctionMinMaxAny.cpp
+// and modified by Doris
+
+#include "vec/aggregate_functions/aggregate_function_min_max.h"
+
+#include "vec/aggregate_functions/aggregate_function_simple_factory.h"
+#include "vec/aggregate_functions/factory_helpers.h"
+#include "vec/aggregate_functions/helpers.h"
+
+namespace doris::vectorized {
+
+/// min, max
+template <template <typename, bool> class AggregateFunctionTemplate, template <typename> class Data>
+static IAggregateFunction* create_aggregate_function_single_value(const String& name,
+ const DataTypes& argument_types,
+ const Array& parameters) {
+ assert_no_parameters(name, parameters);
+ assert_unary(name, argument_types);
+
+ const DataTypePtr& argument_type = argument_types[0];
+
+ WhichDataType which(argument_type);
+#define DISPATCH(TYPE) \
+ if (which.idx == TypeIndex::TYPE) \
+ return new AggregateFunctionTemplate<Data<SingleValueDataFixed<TYPE>>, false>( \
+ argument_type);
+ FOR_NUMERIC_TYPES(DISPATCH)
+#undef DISPATCH
+ if (which.idx == TypeIndex::String) {
+ return new AggregateFunctionTemplate<Data<SingleValueDataString>, false>(argument_type);
+ }
+ if (which.idx == TypeIndex::DateTime || which.idx == TypeIndex::Date) {
+ return new AggregateFunctionTemplate<Data<SingleValueDataFixed<Int64>>, false>(
+ argument_type);
+ }
+ if (which.idx == TypeIndex::Decimal128) {
+ return new AggregateFunctionTemplate<Data<SingleValueDataFixed<DecimalV2Value>>, false>(
+ argument_type);
+ }
+ return nullptr;
+}
+
+AggregateFunctionPtr create_aggregate_function_max(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable) {
+ return AggregateFunctionPtr(
+ create_aggregate_function_single_value<AggregateFunctionsSingleValue,
+ AggregateFunctionMaxData>(name, argument_types,
+ parameters));
+}
+
+AggregateFunctionPtr create_aggregate_function_min(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable) {
+ return AggregateFunctionPtr(
+ create_aggregate_function_single_value<AggregateFunctionsSingleValue,
+ AggregateFunctionMinData>(name, argument_types,
+ parameters));
+}
+
+void register_aggregate_function_minmax(AggregateFunctionSimpleFactory& factory) {
+ factory.register_function("max", create_aggregate_function_max);
+ factory.register_function("min", create_aggregate_function_min);
+}
+
+} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/src/vec/aggregate_functions/aggregate_function_min_max.h b/be/src/vec/aggregate_functions/aggregate_function_min_max.h
new file mode 100644
index 0000000..17d6823
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_min_max.h
@@ -0,0 +1,535 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/AggregateFunctions/AggregateFunctionMinMaxAny.h
+// and modified by Doris
+
+#pragma once
+
+#include "common/logging.h"
+#include "vec/aggregate_functions/aggregate_function.h"
+#include "vec/columns/column_decimal.h"
+#include "vec/columns/column_vector.h"
+#include "vec/common/assert_cast.h"
+#include "vec/io/io_helper.h"
+
+namespace doris::vectorized {
+
+/// For numeric values.
+template <typename T>
+struct SingleValueDataFixed {
+private:
+ using Self = SingleValueDataFixed;
+
+ bool has_value = false; /// We need to remember if at least one value has been passed. This is necessary for AggregateFunctionIf.
+ T value;
+
+public:
+ bool has() const { return has_value; }
+
+ void insert_result_into(IColumn& to) const {
+ if (has())
+ assert_cast<ColumnVector<T>&>(to).get_data().push_back(value);
+ else
+ assert_cast<ColumnVector<T>&>(to).insert_default();
+ }
+
+ void reset() {
+ if (has()) {
+ has_value = false;
+ }
+ }
+
+ void write(BufferWritable& buf) const {
+ write_binary(has(), buf);
+ if (has()) write_binary(value, buf);
+ }
+
+ void read(BufferReadable& buf) {
+ read_binary(has_value, buf);
+ if (has()) read_binary(value, buf);
+ }
+
+ void change(const IColumn& column, size_t row_num, Arena*) {
+ has_value = true;
+ value = assert_cast<const ColumnVector<T>&>(column).get_data()[row_num];
+ }
+
+ /// Assuming to.has()
+ void change(const Self& to, Arena*) {
+ has_value = true;
+ value = to.value;
+ }
+
+ bool change_first_time(const IColumn& column, size_t row_num, Arena* arena) {
+ if (!has()) {
+ change(column, row_num, arena);
+ return true;
+ } else
+ return false;
+ }
+
+ bool change_first_time(const Self& to, Arena* arena) {
+ if (!has() && to.has()) {
+ change(to, arena);
+ return true;
+ } else
+ return false;
+ }
+
+ bool change_every_time(const IColumn& column, size_t row_num, Arena* arena) {
+ change(column, row_num, arena);
+ return true;
+ }
+
+ bool change_every_time(const Self& to, Arena* arena) {
+ if (to.has()) {
+ change(to, arena);
+ return true;
+ } else
+ return false;
+ }
+
+ bool change_if_less(const IColumn& column, size_t row_num, Arena* arena) {
+ if (!has() || assert_cast<const ColumnVector<T>&>(column).get_data()[row_num] < value) {
+ change(column, row_num, arena);
+ return true;
+ } else
+ return false;
+ }
+
+ bool change_if_less(const Self& to, Arena* arena) {
+ if (to.has() && (!has() || to.value < value)) {
+ change(to, arena);
+ return true;
+ } else
+ return false;
+ }
+
+ bool change_if_greater(const IColumn& column, size_t row_num, Arena* arena) {
+ if (!has() || assert_cast<const ColumnVector<T>&>(column).get_data()[row_num] > value) {
+ change(column, row_num, arena);
+ return true;
+ } else
+ return false;
+ }
+
+ bool change_if_greater(const Self& to, Arena* arena) {
+ if (to.has() && (!has() || to.value > value)) {
+ change(to, arena);
+ return true;
+ } else
+ return false;
+ }
+
+ bool is_equal_to(const Self& to) const { return has() && to.value == value; }
+
+ bool is_equal_to(const IColumn& column, size_t row_num) const {
+ return has() && assert_cast<const ColumnVector<T>&>(column).get_data()[row_num] == value;
+ }
+};
+
+/// For numeric values.
+template <>
+struct SingleValueDataFixed<DecimalV2Value> {
+private:
+ using Self = SingleValueDataFixed;
+
+ bool has_value =
+ false; /// We need to remember if at least one value has been passed. This is necessary for AggregateFunctionIf.
+ int128_t value;
+
+public:
+ bool has() const { return has_value; }
+
+ void insert_result_into(IColumn& to) const {
+ if (has()) {
+ DecimalV2Value decimal(value);
+ assert_cast<ColumnDecimal<Decimal128>&>(to).insert_data((const char*)&decimal, 0);
+ } else
+ assert_cast<ColumnDecimal<Decimal128>&>(to).insert_default();
+ }
+
+ void reset() {
+ if (has()) {
+ has_value = false;
+ }
+ }
+
+ void write(BufferWritable& buf) const {
+ write_binary(has(), buf);
+ if (has()) write_binary(value, buf);
+ }
+
+ void read(BufferReadable& buf) {
+ read_binary(has_value, buf);
+ if (has()) read_binary(value, buf);
+ }
+
+ void change(const IColumn& column, size_t row_num, Arena*) {
+ has_value = true;
+ value = assert_cast<const ColumnDecimal<Decimal128>&>(column).get_data()[row_num];
+ }
+
+ /// Assuming to.has()
+ void change(const Self& to, Arena*) {
+ has_value = true;
+ value = to.value;
+ }
+
+ bool change_first_time(const IColumn& column, size_t row_num, Arena* arena) {
+ if (!has()) {
+ change(column, row_num, arena);
+ return true;
+ } else
+ return false;
+ }
+
+ bool change_first_time(const Self& to, Arena* arena) {
+ if (!has() && to.has()) {
+ change(to, arena);
+ return true;
+ } else
+ return false;
+ }
+
+ bool change_every_time(const IColumn& column, size_t row_num, Arena* arena) {
+ change(column, row_num, arena);
+ return true;
+ }
+
+ bool change_every_time(const Self& to, Arena* arena) {
+ if (to.has()) {
+ change(to, arena);
+ return true;
+ } else
+ return false;
+ }
+
+ bool change_if_less(const IColumn& column, size_t row_num, Arena* arena) {
+ if (!has() ||
+ assert_cast<const ColumnDecimal<Decimal128>&>(column).get_data()[row_num] < value) {
+ change(column, row_num, arena);
+ return true;
+ } else
+ return false;
+ }
+
+ bool change_if_less(const Self& to, Arena* arena) {
+ if (to.has() && (!has() || to.value < value)) {
+ change(to, arena);
+ return true;
+ } else
+ return false;
+ }
+
+ bool change_if_greater(const IColumn& column, size_t row_num, Arena* arena) {
+ if (!has() ||
+ assert_cast<const ColumnDecimal<Decimal128>&>(column).get_data()[row_num] > value) {
+ change(column, row_num, arena);
+ return true;
+ } else
+ return false;
+ }
+
+ bool change_if_greater(const Self& to, Arena* arena) {
+ if (to.has() && (!has() || to.value > value)) {
+ change(to, arena);
+ return true;
+ } else
+ return false;
+ }
+
+ bool is_equal_to(const Self& to) const { return has() && to.value == value; }
+
+ bool is_equal_to(const IColumn& column, size_t row_num) const {
+ return has() &&
+ assert_cast<const ColumnDecimal<Decimal128>&>(column).get_data()[row_num] == value;
+ }
+};
+
+/** For strings. Short strings are stored in the object itself, and long strings are allocated separately.
+ * NOTE It could also be suitable for arrays of numbers.
+ */
+struct SingleValueDataString {
+private:
+ using Self = SingleValueDataString;
+
+ Int32 size = -1; /// -1 indicates that there is no value.
+ Int32 capacity = 0; /// power of two or zero
+ char* large_data = nullptr;
+
+public:
+ static constexpr Int32 AUTOMATIC_STORAGE_SIZE = 64;
+ static constexpr Int32 MAX_SMALL_STRING_SIZE =
+ AUTOMATIC_STORAGE_SIZE - sizeof(size) - sizeof(capacity) - sizeof(large_data);
+
+private:
+ char small_data[MAX_SMALL_STRING_SIZE]; /// Including the terminating zero.
+
+public:
+ ~SingleValueDataString() { delete large_data; }
+
+ bool has() const { return size >= 0; }
+
+ const char* get_data() const { return size <= MAX_SMALL_STRING_SIZE ? small_data : large_data; }
+
+ void insert_result_into(IColumn& to) const {
+ if (has())
+ assert_cast<ColumnString&>(to).insert_data(get_data(), size);
+ else
+ assert_cast<ColumnString&>(to).insert_default();
+ }
+
+ void reset() {
+ if (size != -1) {
+ size = -1;
+ capacity = 0;
+ delete large_data;
+ large_data = nullptr;
+ }
+ }
+
+ void write(BufferWritable& buf) const {
+ write_binary(size, buf);
+ if (has()) buf.write(get_data(), size);
+ }
+
+ void read(BufferReadable& buf) {
+ Int32 rhs_size;
+ read_binary(rhs_size, buf);
+
+ if (rhs_size >= 0) {
+ if (rhs_size <= MAX_SMALL_STRING_SIZE) {
+ /// Don't free large_data here.
+
+ size = rhs_size;
+
+ if (size > 0) buf.read(small_data, size);
+ } else {
+ if (capacity < rhs_size) {
+ capacity = static_cast<UInt32>(round_up_to_power_of_two_or_zero(rhs_size));
+ delete large_data;
+ large_data = new char[capacity];
+ }
+
+ size = rhs_size;
+ buf.read(large_data, size);
+ }
+ } else {
+ /// Don't free large_data here.
+ size = rhs_size;
+ }
+ }
+
+ StringRef get_string_ref() const { return StringRef(get_data(), size); }
+
+ /// Assuming to.has()
+ void change_impl(StringRef value, Arena* arena) {
+ Int32 value_size = value.size;
+
+ if (value_size <= MAX_SMALL_STRING_SIZE) {
+ /// Don't free large_data here.
+ size = value_size;
+
+ if (size > 0) {
+ memcpy(small_data, value.data, size);
+ }
+ } else {
+ if (capacity < value_size) {
+ /// Don't free large_data here.
+ capacity = round_up_to_power_of_two_or_zero(value_size);
+ delete large_data;
+ large_data = new char[capacity];
+ }
+
+ size = value_size;
+ memcpy(large_data, value.data, size);
+ }
+ }
+
+ void change(const IColumn& column, size_t row_num, Arena* arena) {
+ change_impl(assert_cast<const ColumnString&>(column).get_data_at(row_num), arena);
+ }
+
+ void change(const Self& to, Arena* arena) { change_impl(to.get_string_ref(), arena); }
+
+ bool change_if_less(const IColumn& column, size_t row_num, Arena* arena) {
+ if (!has() ||
+ assert_cast<const ColumnString&>(column).get_data_at(row_num) < get_string_ref()) {
+ change(column, row_num, arena);
+ return true;
+ } else
+ return false;
+ }
+
+ bool change_if_greater(const IColumn& column, size_t row_num, Arena* arena) {
+ if (!has() ||
+ assert_cast<const ColumnString&>(column).get_data_at(row_num) > get_string_ref()) {
+ change(column, row_num, arena);
+ return true;
+ } else
+ return false;
+ }
+
+ bool change_first_time(const IColumn& column, size_t row_num, Arena* arena) {
+ if (!has()) {
+ change(column, row_num, arena);
+ return true;
+ } else
+ return false;
+ }
+
+ bool change_first_time(const Self& to, Arena* arena) {
+ if (!has() && to.has()) {
+ change(to, arena);
+ return true;
+ } else
+ return false;
+ }
+
+ bool change_every_time(const IColumn& column, size_t row_num, Arena* arena) {
+ change(column, row_num, arena);
+ return true;
+ }
+
+ bool change_every_time(const Self& to, Arena* arena) {
+ if (to.has()) {
+ change(to, arena);
+ return true;
+ } else
+ return false;
+ }
+
+ bool change_if_less(const Self& to, Arena* arena) {
+ if (to.has() && (!has() || to.get_string_ref() < get_string_ref())) {
+ change(to, arena);
+ return true;
+ } else
+ return false;
+ }
+
+ bool change_if_greater(const Self& to, Arena* arena) {
+ if (to.has() && (!has() || to.get_string_ref() > get_string_ref())) {
+ change(to, arena);
+ return true;
+ } else
+ return false;
+ }
+
+ bool is_equal_to(const Self& to) const {
+ return has() && to.get_string_ref() == get_string_ref();
+ }
+
+ bool is_equal_to(const IColumn& column, size_t row_num) const { return false; }
+};
+
+template <typename Data>
+struct AggregateFunctionMaxData : Data {
+ using Self = AggregateFunctionMaxData;
+
+ bool change_if_better(const IColumn& column, size_t row_num, Arena* arena) {
+ return this->change_if_greater(column, row_num, arena);
+ }
+ bool change_if_better(const Self& to, Arena* arena) {
+ return this->change_if_greater(to, arena);
+ }
+
+ static const char* name() { return "max"; }
+};
+
+template <typename Data>
+struct AggregateFunctionMinData : Data {
+ using Self = AggregateFunctionMinData;
+
+ bool change_if_better(const IColumn& column, size_t row_num, Arena* arena) {
+ return this->change_if_less(column, row_num, arena);
+ }
+ bool change_if_better(const Self& to, Arena* arena) { return this->change_if_less(to, arena); }
+
+ static const char* name() { return "min"; }
+};
+
+template <typename Data, bool AllocatesMemoryInArena>
+class AggregateFunctionsSingleValue final
+ : public IAggregateFunctionDataHelper<
+ Data, AggregateFunctionsSingleValue<Data, AllocatesMemoryInArena>> {
+private:
+ DataTypePtr& type;
+
+public:
+ AggregateFunctionsSingleValue(const DataTypePtr& type_)
+ : IAggregateFunctionDataHelper<
+ Data, AggregateFunctionsSingleValue<Data, AllocatesMemoryInArena>>({type_},
+ {}),
+ type(this->argument_types[0]) {
+ if (StringRef(Data::name()) == StringRef("min") ||
+ StringRef(Data::name()) == StringRef("max")) {
+ if (!type->is_comparable()) {
+ LOG(FATAL) << fmt::format(
+ "Illegal type {} of argument of aggregate function {} because the values "
+ "of that data type are not comparable",
+ type->get_name(), get_name());
+ }
+ }
+ }
+
+ String get_name() const override { return Data::name(); }
+
+ DataTypePtr get_return_type() const override { return type; }
+
+ void add(AggregateDataPtr __restrict place, const IColumn** columns, size_t row_num,
+ Arena* arena) const override {
+ this->data(place).change_if_better(*columns[0], row_num, arena);
+ }
+
+ void reset(AggregateDataPtr place) const override {
+ this->data(place).reset();
+ }
+
+ void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena* arena) const override {
+ this->data(place).change_if_better(this->data(rhs), arena);
+ }
+
+ void serialize(ConstAggregateDataPtr __restrict place, BufferWritable& buf) const override {
+ this->data(place).write(buf);
+ }
+
+ void deserialize(AggregateDataPtr __restrict place, BufferReadable& buf, Arena*) const override {
+ this->data(place).read(buf);
+ }
+
+ bool allocates_memory_in_arena() const override { return AllocatesMemoryInArena; }
+
+ void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override {
+ this->data(place).insert_result_into(to);
+ }
+
+ const char* get_header_file_path() const override { return __FILE__; }
+};
+
+AggregateFunctionPtr create_aggregate_function_max(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable);
+
+AggregateFunctionPtr create_aggregate_function_min(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable);
+
+} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/src/vec/aggregate_functions/aggregate_function_nothing.h b/be/src/vec/aggregate_functions/aggregate_function_nothing.h
new file mode 100644
index 0000000..4c7b193
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_nothing.h
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/AggregateFunctions/AggregateFunctionNothing.h
+// and modified by Doris
+
+#pragma once
+
+#include "vec/aggregate_functions/aggregate_function.h"
+#include "vec/columns/column.h"
+#include "vec/data_types/data_type_nothing.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/io/io_helper.h"
+
+namespace doris::vectorized {
+
+/** Aggregate function that takes arbitrary number of arbitrary arguments and does nothing.
+ */
+class AggregateFunctionNothing final : public IAggregateFunctionHelper<AggregateFunctionNothing> {
+public:
+ AggregateFunctionNothing(const DataTypes& arguments, const Array& params)
+ : IAggregateFunctionHelper<AggregateFunctionNothing>(arguments, params) {}
+
+ String get_name() const override { return "nothing"; }
+
+ DataTypePtr get_return_type() const override {
+ return std::make_shared<DataTypeNullable>(std::make_shared<DataTypeNothing>());
+ }
+
+ void create(AggregateDataPtr) const override {}
+
+ void destroy(AggregateDataPtr) const noexcept override {}
+
+ bool has_trivial_destructor() const override { return true; }
+
+ size_t size_of_data() const override { return 0; }
+
+ size_t align_of_data() const override { return 1; }
+
+ void add(AggregateDataPtr, const IColumn**, size_t, Arena*) const override {}
+
+ void reset(AggregateDataPtr place) const override {}
+
+ void merge(AggregateDataPtr, ConstAggregateDataPtr, Arena*) const override {}
+
+ void serialize(ConstAggregateDataPtr, BufferWritable& buf) const override {}
+
+ void deserialize(AggregateDataPtr, BufferReadable& buf, Arena*) const override {}
+
+ void insert_result_into(ConstAggregateDataPtr, IColumn& to) const override {
+ to.insert_default();
+ }
+
+ const char* get_header_file_path() const override { return __FILE__; }
+};
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/aggregate_functions/aggregate_function_null.cpp b/be/src/vec/aggregate_functions/aggregate_function_null.cpp
new file mode 100644
index 0000000..e28287f
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_null.cpp
@@ -0,0 +1,92 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/AggregateFunctions/AggregateFunctionNull.cpp
+// and modified by Doris
+
+#include "vec/aggregate_functions/aggregate_function_null.h"
+
+#include "common/logging.h"
+#include "vec/aggregate_functions/aggregate_function.h"
+#include "vec/aggregate_functions/aggregate_function_combinator.h"
+#include "vec/aggregate_functions/aggregate_function_count.h"
+#include "vec/aggregate_functions/aggregate_function_nothing.h"
+#include "vec/aggregate_functions/aggregate_function_simple_factory.h"
+#include "vec/data_types/data_type_nullable.h"
+
+namespace doris::vectorized {
+
+class AggregateFunctionCombinatorNull final : public IAggregateFunctionCombinator {
+public:
+ String get_name() const override { return "Null"; }
+
+ bool is_for_internal_usage_only() const override { return true; }
+
+ DataTypes transform_arguments(const DataTypes& arguments) const override {
+ size_t size = arguments.size();
+ DataTypes res(size);
+ for (size_t i = 0; i < size; ++i) res[i] = remove_nullable(arguments[i]);
+ return res;
+ }
+
+ AggregateFunctionPtr transform_aggregate_function(const AggregateFunctionPtr& nested_function,
+ const DataTypes& arguments,
+ const Array& params,
+ const bool result_is_nullable) const override {
+ if (nested_function == nullptr) return nullptr;
+
+ bool has_null_types = false;
+ for (const auto& arg_type : arguments) {
+ if (arg_type->only_null()) {
+ has_null_types = true;
+ break;
+ }
+ }
+
+ if (has_null_types) return std::make_shared<AggregateFunctionNothing>(arguments, params);
+
+ if (arguments.size() == 1) {
+ if (result_is_nullable)
+ return std::make_shared<AggregateFunctionNullUnary<true>>(nested_function,
+ arguments, params);
+ else
+ return std::make_shared<AggregateFunctionNullUnary<false>>(nested_function,
+ arguments, params);
+ } else {
+ if (result_is_nullable)
+ return std::make_shared<AggregateFunctionNullVariadic<true>>(nested_function,
+ arguments, params);
+ else
+ return std::make_shared<AggregateFunctionNullVariadic<false>>(nested_function,
+ arguments, params);
+ }
+ }
+};
+
+void register_aggregate_function_combinator_null(AggregateFunctionSimpleFactory& factory) {
+ // factory.registerCombinator(std::make_shared<AggregateFunctionCombinatorNull>());
+ AggregateFunctionCreator creator = [&](const std::string& name, const DataTypes& types,
+ const Array& params, const bool result_is_nullable) {
+ auto function_combinator = std::make_shared<AggregateFunctionCombinatorNull>();
+ auto transform_arguments = function_combinator->transform_arguments(types);
+ auto nested_function = factory.get(name, transform_arguments, params);
+ return function_combinator->transform_aggregate_function(nested_function, types, params, result_is_nullable);
+ };
+ factory.register_nullable_function_combinator(creator);
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/aggregate_functions/aggregate_function_null.h b/be/src/vec/aggregate_functions/aggregate_function_null.h
new file mode 100644
index 0000000..4c61e2b
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_null.h
@@ -0,0 +1,291 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/AggregateFunctions/AggregateFunctionNull.h
+// and modified by Doris
+
+#pragma once
+
+#include <array>
+
+#include "common/logging.h"
+#include "common/status.h"
+#include "vec/aggregate_functions/aggregate_function.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/common/assert_cast.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/io/io_helper.h"
+
+namespace doris::vectorized {
+
+/// This class implements a wrapper around an aggregate function. Despite its name,
+/// this is an adapter. It is used to handle aggregate functions that are called with
+/// at least one nullable argument. It implements the logic according to which any
+/// row that contains at least one NULL is skipped.
+
+/// If all rows had NULL, the behaviour is determined by "result_is_nullable" template parameter.
+/// true - return NULL; false - return value from empty aggregation state of nested function.
+
+template <bool result_is_nullable, typename Derived>
+class AggregateFunctionNullBase : public IAggregateFunctionHelper<Derived> {
+protected:
+ AggregateFunctionPtr nested_function;
+ size_t prefix_size;
+
+ /** In addition to data for nested aggregate function, we keep a flag
+ * indicating - was there at least one non-NULL value accumulated.
+ * In case of no not-NULL values, the function will return NULL.
+ *
+ * We use prefix_size bytes for flag to satisfy the alignment requirement of nested state.
+ */
+
+ AggregateDataPtr nested_place(AggregateDataPtr __restrict place) const noexcept {
+ return place + prefix_size;
+ }
+
+ ConstAggregateDataPtr nested_place(ConstAggregateDataPtr __restrict place) const noexcept {
+ return place + prefix_size;
+ }
+
+ static void init_flag(AggregateDataPtr __restrict place) noexcept {
+ if constexpr (result_is_nullable)
+ place[0] = 0;
+ }
+
+ static void set_flag(AggregateDataPtr __restrict place) noexcept {
+ if constexpr (result_is_nullable)
+ place[0] = 1;
+ }
+
+ static bool get_flag(ConstAggregateDataPtr __restrict place) noexcept {
+ return result_is_nullable ? place[0] : 1;
+ }
+
+public:
+ AggregateFunctionNullBase(AggregateFunctionPtr nested_function_, const DataTypes& arguments,
+ const Array& params)
+ : IAggregateFunctionHelper<Derived>(arguments, params),
+ nested_function {nested_function_} {
+ if (result_is_nullable)
+ prefix_size = nested_function->align_of_data();
+ else
+ prefix_size = 0;
+ }
+
+ String get_name() const override {
+ /// This is just a wrapper. The function for Nullable arguments is named the same as the nested function itself.
+ return nested_function->get_name();
+ }
+
+ DataTypePtr get_return_type() const override {
+ return result_is_nullable ? make_nullable(nested_function->get_return_type())
+ : nested_function->get_return_type();
+ }
+
+ void create(AggregateDataPtr __restrict place) const override {
+ init_flag(place);
+ nested_function->create(nested_place(place));
+ }
+
+ void destroy(AggregateDataPtr __restrict place) const noexcept override {
+ nested_function->destroy(nested_place(place));
+ }
+ void reset(AggregateDataPtr place) const override {
+ init_flag(place);
+ nested_function->reset(nested_place(place));
+ }
+
+ bool has_trivial_destructor() const override {
+ return nested_function->has_trivial_destructor();
+ }
+
+ size_t size_of_data() const override { return prefix_size + nested_function->size_of_data(); }
+
+ size_t align_of_data() const override { return nested_function->align_of_data(); }
+
+ void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena* arena) const override {
+ if (result_is_nullable && get_flag(rhs)) set_flag(place);
+
+ nested_function->merge(nested_place(place), nested_place(rhs), arena);
+ }
+
+ void serialize(ConstAggregateDataPtr __restrict place, BufferWritable& buf) const override {
+ bool flag = get_flag(place);
+ if (result_is_nullable) write_binary(flag, buf);
+ if (flag) {
+ nested_function->serialize(nested_place(place), buf);
+ }
+ }
+
+ void deserialize(AggregateDataPtr __restrict place, BufferReadable& buf, Arena* arena) const override {
+ bool flag = true;
+ if (result_is_nullable) read_binary(flag, buf);
+ if (flag) {
+ set_flag(place);
+ nested_function->deserialize(nested_place(place), buf, arena);
+ }
+ }
+
+ void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override {
+ if constexpr (result_is_nullable) {
+ ColumnNullable& to_concrete = assert_cast<ColumnNullable&>(to);
+ if (get_flag(place)) {
+ nested_function->insert_result_into(nested_place(place),
+ to_concrete.get_nested_column());
+ to_concrete.get_null_map_data().push_back(0);
+ } else {
+ to_concrete.insert_default();
+ }
+ } else {
+ nested_function->insert_result_into(nested_place(place), to);
+ }
+ }
+
+ bool allocates_memory_in_arena() const override {
+ return nested_function->allocates_memory_in_arena();
+ }
+
+ bool is_state() const override { return nested_function->is_state(); }
+
+ const char* get_header_file_path() const override { return __FILE__; }
+};
+
+/** There are two cases: for single argument and variadic.
+ * Code for single argument is much more efficient.
+ */
+template <bool result_is_nullable>
+class AggregateFunctionNullUnary final
+ : public AggregateFunctionNullBase<result_is_nullable,
+ AggregateFunctionNullUnary<result_is_nullable>> {
+public:
+ AggregateFunctionNullUnary(AggregateFunctionPtr nested_function_, const DataTypes& arguments,
+ const Array& params)
+ : AggregateFunctionNullBase<result_is_nullable,
+ AggregateFunctionNullUnary<result_is_nullable>>(
+ std::move(nested_function_), arguments, params) {}
+
+ void add(AggregateDataPtr __restrict place, const IColumn** columns, size_t row_num,
+ Arena* arena) const override {
+ const ColumnNullable* column = assert_cast<const ColumnNullable*>(columns[0]);
+ if (!column->is_null_at(row_num)) {
+ this->set_flag(place);
+ const IColumn* nested_column = &column->get_nested_column();
+ this->nested_function->add(this->nested_place(place), &nested_column, row_num, arena);
+ }
+ }
+
+ void add_batch_single_place(size_t batch_size, AggregateDataPtr place, const IColumn** columns,
+ Arena* arena) const override {
+ const ColumnNullable* column = assert_cast<const ColumnNullable*>(columns[0]);
+ bool has_null = column->has_null();
+
+ if (has_null) {
+ for (size_t i = 0; i < batch_size; ++i) {
+ if (!column->is_null_at(i)) {
+ this->set_flag(place);
+ this->add(place, columns, i, arena);
+ }
+ }
+ } else {
+ this->set_flag(place);
+ const IColumn* nested_column = &column->get_nested_column();
+ this->nested_function->add_batch_single_place(batch_size, this->nested_place(place),
+ &nested_column, arena);
+ }
+ }
+
+ void add_batch_range(size_t batch_begin, size_t batch_end, AggregateDataPtr place,
+ const IColumn** columns, Arena* arena, bool has_null) override {
+ const ColumnNullable* column = assert_cast<const ColumnNullable*>(columns[0]);
+
+ if (has_null) {
+ for (size_t i = batch_begin; i <= batch_end; ++i) {
+ if (!column->is_null_at(i)) {
+ this->set_flag(place);
+ this->add(place, columns, i, arena);
+ }
+ }
+ } else {
+ this->set_flag(place);
+ const IColumn* nested_column = &column->get_nested_column();
+ this->nested_function->add_batch_range(
+ batch_begin, batch_end, this->nested_place(place), &nested_column, arena);
+ }
+ }
+};
+
+template <bool result_is_nullable>
+class AggregateFunctionNullVariadic final
+ : public AggregateFunctionNullBase<result_is_nullable,
+ AggregateFunctionNullVariadic<result_is_nullable>> {
+public:
+ AggregateFunctionNullVariadic(AggregateFunctionPtr nested_function_, const DataTypes& arguments,
+ const Array& params)
+ : AggregateFunctionNullBase<result_is_nullable,
+ AggregateFunctionNullVariadic<result_is_nullable>>(
+ std::move(nested_function_), arguments, params),
+ number_of_arguments(arguments.size()) {
+ if (number_of_arguments == 1) {
+ LOG(FATAL)
+ << "Logical error: single argument is passed to AggregateFunctionNullVariadic";
+ }
+
+ if (number_of_arguments > MAX_ARGS) {
+ LOG(FATAL) << fmt::format(
+ "Maximum number of arguments for aggregate function with Nullable types is {}",
+ size_t(MAX_ARGS));
+ }
+
+ for (size_t i = 0; i < number_of_arguments; ++i)
+ is_nullable[i] = arguments[i]->is_nullable();
+ }
+
+ void add(AggregateDataPtr __restrict place, const IColumn** columns, size_t row_num,
+ Arena* arena) const override {
+ /// This container stores the columns we really pass to the nested function.
+ const IColumn* nested_columns[number_of_arguments];
+
+ for (size_t i = 0; i < number_of_arguments; ++i) {
+ if (is_nullable[i]) {
+ const ColumnNullable& nullable_col =
+ assert_cast<const ColumnNullable&>(*columns[i]);
+ if (nullable_col.is_null_at(row_num)) {
+ /// If at least one column has a null value in the current row,
+ /// we don't process this row.
+ return;
+ }
+ nested_columns[i] = &nullable_col.get_nested_column();
+ } else
+ nested_columns[i] = columns[i];
+ }
+
+ this->set_flag(place);
+ this->nested_function->add(this->nested_place(place), nested_columns, row_num, arena);
+ }
+
+ bool allocates_memory_in_arena() const override {
+ return this->nested_function->allocates_memory_in_arena();
+ }
+
+private:
+ enum { MAX_ARGS = 8 };
+ size_t number_of_arguments = 0;
+ std::array<char, MAX_ARGS>
+ is_nullable; /// Plain array is better than std::vector due to one indirection less.
+};
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/aggregate_functions/aggregate_function_reader.cpp b/be/src/vec/aggregate_functions/aggregate_function_reader.cpp
new file mode 100644
index 0000000..9a24ac5
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_reader.cpp
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/aggregate_functions/aggregate_function_reader.h"
+
+namespace doris::vectorized {
+
+// auto spread at nullable condition, null value do not participate aggregate
+void register_aggregate_function_reader(AggregateFunctionSimpleFactory& factory) {
+ // add a suffix to the function name here to distinguish special functions of agg reader
+ auto register_function_reader = [&](const std::string& name,
+ const AggregateFunctionCreator& creator,
+ bool nullable = false) {
+ factory.register_function(name + agg_reader_suffix, creator, nullable);
+ };
+
+ register_function_reader("sum", create_aggregate_function_sum_reader);
+ register_function_reader("max", create_aggregate_function_max);
+ register_function_reader("min", create_aggregate_function_min);
+ register_function_reader("replace_if_not_null", create_aggregate_function_replace_if_not_null);
+ register_function_reader("bitmap_union", create_aggregate_function_bitmap_union);
+ register_function_reader("hll_union", create_aggregate_function_HLL_union);
+}
+
+void register_aggregate_function_reader_no_spread(AggregateFunctionSimpleFactory& factory) {
+ auto register_function_reader = [&](const std::string& name,
+ const AggregateFunctionCreator& creator,
+ bool nullable = false) {
+ factory.register_function(name + agg_reader_suffix, creator, nullable);
+ };
+
+ register_function_reader("replace", create_aggregate_function_replace, false);
+ register_function_reader("replace", create_aggregate_function_replace_nullable, true);
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/olap/rowset/segment_v2/empty_segment_iterator.h b/be/src/vec/aggregate_functions/aggregate_function_reader.h
similarity index 54%
copy from be/src/olap/rowset/segment_v2/empty_segment_iterator.h
copy to be/src/vec/aggregate_functions/aggregate_function_reader.h
index cb6c48c..f44be5e 100644
--- a/be/src/olap/rowset/segment_v2/empty_segment_iterator.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_reader.h
@@ -17,25 +17,19 @@
#pragma once
-#include "common/status.h"
-#include "olap/iterators.h"
-#include "olap/rowset/segment_v2/segment.h"
-#include "olap/schema.h"
+#include "vec/aggregate_functions/aggregate_function_bitmap.h"
+#include "vec/aggregate_functions/aggregate_function_hll_union_agg.h"
+#include "vec/aggregate_functions/aggregate_function_min_max.h"
+#include "vec/aggregate_functions/aggregate_function_simple_factory.h"
+#include "vec/aggregate_functions/aggregate_function_sum.h"
+#include "vec/aggregate_functions/aggregate_function_window.h"
-namespace doris {
-namespace segment_v2 {
+namespace doris::vectorized {
-class EmptySegmentIterator : public RowwiseIterator {
-public:
- explicit EmptySegmentIterator(const Schema& schema);
- ~EmptySegmentIterator() override {}
- Status init(const StorageReadOptions& opts) override { return Status::OK(); }
- const Schema& schema() const override { return _schema; }
- Status next_batch(RowBlockV2* row_block) override;
+static const std::string agg_reader_suffix = "_reader";
-private:
- Schema _schema;
-};
+void register_aggregate_function_reader(AggregateFunctionSimpleFactory& factory);
-} // namespace segment_v2
-} // namespace doris
\ No newline at end of file
+void register_aggregate_function_reader_no_spread(AggregateFunctionSimpleFactory& factory);
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp b/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp
new file mode 100644
index 0000000..8a1995b
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/AggregateFunctions/AggregateFunctionFactory.cpp
+// and modified by Doris
+
+#include "vec/aggregate_functions/aggregate_function_simple_factory.h"
+
+#include "vec/aggregate_functions/aggregate_function_reader.h"
+
+namespace doris::vectorized {
+class AggregateFunctionSimpleFactory;
+void register_aggregate_function_sum(AggregateFunctionSimpleFactory& factory);
+void register_aggregate_function_combinator_null(AggregateFunctionSimpleFactory& factory);
+void register_aggregate_function_minmax(AggregateFunctionSimpleFactory& factory);
+void register_aggregate_function_avg(AggregateFunctionSimpleFactory& factory);
+void register_aggregate_function_count(AggregateFunctionSimpleFactory& factory);
+void register_aggregate_function_HLL_union_agg(AggregateFunctionSimpleFactory& factory);
+void register_aggregate_function_uniq(AggregateFunctionSimpleFactory& factory);
+void register_aggregate_function_combinator_distinct(AggregateFunctionSimpleFactory& factory);
+void register_aggregate_function_bitmap(AggregateFunctionSimpleFactory& factory);
+void register_aggregate_function_window_rank(AggregateFunctionSimpleFactory& factory);
+void register_aggregate_function_window_lead_lag(AggregateFunctionSimpleFactory& factory);
+AggregateFunctionSimpleFactory& AggregateFunctionSimpleFactory::instance() {
+ static std::once_flag oc;
+ static AggregateFunctionSimpleFactory instance;
+ std::call_once(oc, [&]() {
+ register_aggregate_function_sum(instance);
+ register_aggregate_function_minmax(instance);
+ register_aggregate_function_avg(instance);
+ register_aggregate_function_count(instance);
+ register_aggregate_function_uniq(instance);
+ register_aggregate_function_bitmap(instance);
+ register_aggregate_function_combinator_distinct(instance);
+ register_aggregate_function_HLL_union_agg(instance);
+ register_aggregate_function_reader(instance); // register aggregate function for agg reader
+ register_aggregate_function_window_rank(instance);
+
+ // if you only register function with no nullable, and wants to add nullable automatically, you should place function above this line
+ register_aggregate_function_combinator_null(instance);
+
+ register_aggregate_function_reader_no_spread(instance);
+ register_aggregate_function_window_lead_lag(instance);
+ });
+ return instance;
+}
+
+} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/src/vec/aggregate_functions/aggregate_function_simple_factory.h b/be/src/vec/aggregate_functions/aggregate_function_simple_factory.h
new file mode 100644
index 0000000..d12c27a
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_simple_factory.h
@@ -0,0 +1,104 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/AggregateFunctions/AggregateFunctionFactory.h
+// and modified by Doris
+
+#pragma once
+
+#include <functional>
+#include <iostream>
+#include <memory>
+#include <mutex>
+#include <unordered_map>
+
+#include "vec/aggregate_functions/aggregate_function.h"
+#include "vec/core/field.h"
+#include "vec/data_types/data_type.h"
+
+namespace doris::vectorized {
+using DataTypePtr = std::shared_ptr<const IDataType>;
+using DataTypes = std::vector<DataTypePtr>;
+using AggregateFunctionCreator = std::function<AggregateFunctionPtr(
+ const std::string&, const DataTypes&, const Array&, const bool)>;
+
+class AggregateFunctionSimpleFactory {
+public:
+ using Creator = AggregateFunctionCreator;
+
+private:
+ using AggregateFunctions = std::unordered_map<std::string, Creator>;
+
+ AggregateFunctions aggregate_functions;
+ AggregateFunctions nullable_aggregate_functions;
+
+public:
+ void register_nullable_function_combinator(const Creator& creator) {
+ for (const auto& entity : aggregate_functions) {
+ if (nullable_aggregate_functions.find(entity.first) ==
+ nullable_aggregate_functions.end()) {
+ nullable_aggregate_functions[entity.first] = creator;
+ }
+ }
+ }
+
+ void register_distinct_function_combinator(const Creator& creator, const std::string& prefix) {
+ std::vector<std::string> need_insert;
+ for (const auto& entity : aggregate_functions) {
+ std::string target_value = prefix + entity.first;
+ if (aggregate_functions.find(target_value) == aggregate_functions.end()) {
+ need_insert.emplace_back(std::move(target_value));
+ }
+ }
+ for (const auto& function_name : need_insert) {
+ aggregate_functions[function_name] = creator;
+ }
+ }
+
+ AggregateFunctionPtr get(const std::string& name, const DataTypes& argument_types,
+ const Array& parameters, const bool result_is_nullable = false) {
+ bool nullable = false;
+ for (const auto& type : argument_types) {
+ if (type->is_nullable()) {
+ nullable = true;
+ }
+ }
+ if (nullable) {
+ return nullable_aggregate_functions.find(name) == nullable_aggregate_functions.end()
+ ? nullptr
+ : nullable_aggregate_functions[name](name, argument_types, parameters,
+ result_is_nullable);
+ } else {
+ return aggregate_functions.find(name) == aggregate_functions.end()
+ ? nullptr
+ : aggregate_functions[name](name, argument_types, parameters,
+ result_is_nullable);
+ }
+ }
+
+ void register_function(const std::string& name, const Creator& creator, bool nullable = false) {
+ if (nullable) {
+ nullable_aggregate_functions[name] = creator;
+ } else {
+ aggregate_functions[name] = creator;
+ }
+ }
+
+public:
+ static AggregateFunctionSimpleFactory& instance();
+};
+}; // namespace doris::vectorized
diff --git a/be/src/vec/aggregate_functions/aggregate_function_sum.cpp b/be/src/vec/aggregate_functions/aggregate_function_sum.cpp
new file mode 100644
index 0000000..f7127e7
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_sum.cpp
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/AggregateFunctions/AggregateFunctionSum.cpp
+// and modified by Doris
+
+#include "vec/aggregate_functions/aggregate_function_sum.h"
+
+#include <fmt/format.h>
+
+#include "common/logging.h"
+#include "vec/aggregate_functions/aggregate_function_simple_factory.h"
+#include "vec/aggregate_functions/helpers.h"
+
+namespace doris::vectorized {
+
+template <typename T>
+struct SumSimple {
+ /// @note It uses slow Decimal128 (cause we need such a variant). sumWithOverflow is faster for Decimal32/64
+ using ResultType = std::conditional_t<IsDecimalNumber<T>, Decimal128, NearestFieldType<T>>;
+ // using ResultType = NearestFieldType<T>;
+ using AggregateDataType = AggregateFunctionSumData<ResultType>;
+ using Function = AggregateFunctionSum<T, ResultType, AggregateDataType>;
+};
+
+template <typename T>
+using AggregateFunctionSumSimple = typename SumSimple<T>::Function;
+
+template <template <typename> class Function>
+AggregateFunctionPtr create_aggregate_function_sum(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable) {
+ // assert_no_parameters(name, parameters);
+ // assert_unary(name, argument_types);
+
+ AggregateFunctionPtr res;
+ DataTypePtr data_type = argument_types[0];
+ if (is_decimal(data_type))
+ res.reset(create_with_decimal_type<Function>(*data_type, *data_type, argument_types));
+ else
+ res.reset(create_with_numeric_type<Function>(*data_type, argument_types));
+
+ if (!res) {
+ LOG(WARNING) << fmt::format("Illegal type {} of argument for aggregate function {}",
+ argument_types[0]->get_name(), name);
+ }
+ return res;
+}
+
+// do not level up return type for agg reader
+template <typename T>
+struct SumSimpleReader {
+ using ResultType = T;
+ using AggregateDataType = AggregateFunctionSumData<ResultType>;
+ using Function = AggregateFunctionSum<T, ResultType, AggregateDataType>;
+};
+
+template <typename T>
+using AggregateFunctionSumSimpleReader = typename SumSimpleReader<T>::Function;
+
+AggregateFunctionPtr create_aggregate_function_sum_reader(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable) {
+ return create_aggregate_function_sum<AggregateFunctionSumSimpleReader>(
+ name, argument_types, parameters, result_is_nullable);
+}
+
+void register_aggregate_function_sum(AggregateFunctionSimpleFactory& factory) {
+ factory.register_function("sum", create_aggregate_function_sum<AggregateFunctionSumSimple>);
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/aggregate_functions/aggregate_function_sum.h b/be/src/vec/aggregate_functions/aggregate_function_sum.h
new file mode 100644
index 0000000..402af36
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_sum.h
@@ -0,0 +1,118 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/AggregateFunctions/AggregateFunctionSum.h
+// and modified by Doris
+
+#pragma once
+
+#include <istream>
+#include <ostream>
+#include <type_traits>
+
+#include "vec/aggregate_functions/aggregate_function.h"
+#include "vec/columns/column_vector.h"
+#include "vec/data_types/data_type_decimal.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/io/io_helper.h"
+
+namespace doris::vectorized {
+
+template <typename T>
+struct AggregateFunctionSumData {
+ T sum {};
+
+ void add(T value) { sum += value; }
+
+ void merge(const AggregateFunctionSumData& rhs) { sum += rhs.sum; }
+
+ void write(BufferWritable& buf) const { write_binary(sum, buf); }
+
+ void read(BufferReadable& buf) { read_binary(sum, buf); }
+
+ T get() const { return sum; }
+};
+
+/// Counts the sum of the numbers.
+template <typename T, typename TResult, typename Data>
+class AggregateFunctionSum final
+ : public IAggregateFunctionDataHelper<Data, AggregateFunctionSum<T, TResult, Data>> {
+public:
+ using ResultDataType = std::conditional_t<IsDecimalNumber<T>, DataTypeDecimal<TResult>,
+ DataTypeNumber<TResult>>;
+ using ColVecType = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<T>, ColumnVector<T>>;
+ using ColVecResult =
+ std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<TResult>, ColumnVector<TResult>>;
+
+ String get_name() const override { return "sum"; }
+
+ AggregateFunctionSum(const DataTypes& argument_types_)
+ : IAggregateFunctionDataHelper<Data, AggregateFunctionSum<T, TResult, Data>>(
+ argument_types_, {}),
+ scale(0) {}
+
+ AggregateFunctionSum(const IDataType& data_type, const DataTypes& argument_types_)
+ : IAggregateFunctionDataHelper<Data, AggregateFunctionSum<T, TResult, Data>>(
+ argument_types_, {}),
+ scale(get_decimal_scale(data_type)) {}
+
+ DataTypePtr get_return_type() const override {
+ if constexpr (IsDecimalNumber<T>)
+ return std::make_shared<ResultDataType>(ResultDataType::max_precision(), scale);
+ else
+ return std::make_shared<ResultDataType>();
+ }
+
+ void add(AggregateDataPtr __restrict place, const IColumn** columns, size_t row_num,
+ Arena*) const override {
+ const auto& column = static_cast<const ColVecType&>(*columns[0]);
+ this->data(place).add(column.get_data()[row_num]);
+ }
+
+ void reset(AggregateDataPtr place) const override {
+ this->data(place).sum = {};
+ }
+
+ void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena*) const override {
+ this->data(place).merge(this->data(rhs));
+ }
+
+ void serialize(ConstAggregateDataPtr __restrict place, BufferWritable& buf) const override {
+ this->data(place).write(buf);
+ }
+
+ void deserialize(AggregateDataPtr __restrict place, BufferReadable& buf, Arena*) const override {
+ this->data(place).read(buf);
+ }
+
+ void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override {
+ auto& column = static_cast<ColVecResult&>(to);
+ column.get_data().push_back(this->data(place).get());
+ }
+
+ const char* get_header_file_path() const override { return __FILE__; }
+
+private:
+ UInt32 scale;
+};
+
+AggregateFunctionPtr create_aggregate_function_sum_reader(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable);
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/aggregate_functions/aggregate_function_uniq.cpp b/be/src/vec/aggregate_functions/aggregate_function_uniq.cpp
new file mode 100644
index 0000000..0258858
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_uniq.cpp
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/AggregateFunctions/AggregateFunctionUniq.cpp
+// and modified by Doris
+
+#include "vec/aggregate_functions/aggregate_function_uniq.h"
+
+#include "common/logging.h"
+#include "vec/aggregate_functions/aggregate_function_simple_factory.h"
+#include "vec/aggregate_functions/factory_helpers.h"
+#include "vec/aggregate_functions/helpers.h"
+#include "vec/data_types/data_type_string.h"
+
+namespace doris::vectorized {
+
+template <template <typename> class Data, typename DataForVariadic>
+AggregateFunctionPtr create_aggregate_function_uniq(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& params,
+ const bool result_is_nullable) {
+ assert_no_parameters(name, params);
+
+ if (argument_types.empty()) {
+ LOG(WARNING) << "Incorrect number of arguments for aggregate function " << name;
+ return nullptr;
+ }
+
+ if (argument_types.size() == 1) {
+ const IDataType& argument_type = *argument_types[0];
+
+ AggregateFunctionPtr res(create_with_numeric_type<AggregateFunctionUniq, Data>(
+ *argument_types[0], argument_types));
+
+ WhichDataType which(argument_type);
+ // TODO: DateType
+ if (res)
+ return res;
+ else if (which.is_decimal())
+ return std::make_shared<AggregateFunctionUniq<Decimal128, Data<Int128>>>(argument_types);
+ else if (which.is_string_or_fixed_string())
+ return std::make_shared<AggregateFunctionUniq<String, Data<String>>>(argument_types);
+ }
+
+ return nullptr;
+}
+
+void register_aggregate_function_uniq(AggregateFunctionSimpleFactory& factory) {
+ AggregateFunctionCreator creator =
+ create_aggregate_function_uniq<AggregateFunctionUniqExactData,
+ AggregateFunctionUniqExactData<String>>;
+ factory.register_function("multi_distinct_count", creator);
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/aggregate_functions/aggregate_function_uniq.h b/be/src/vec/aggregate_functions/aggregate_function_uniq.h
new file mode 100644
index 0000000..ea32404
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_uniq.h
@@ -0,0 +1,131 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/AggregateFunctions/AggregateFunctionUniq.h
+// and modified by Doris
+
+#pragma once
+
+#include <type_traits>
+
+#include "gutil/hash/city.h"
+
+#include "vec/aggregate_functions/aggregate_function.h"
+#include "vec/columns/column_decimal.h"
+#include "vec/common/aggregation_common.h"
+#include "vec/common/assert_cast.h"
+#include "vec/common/bit_cast.h"
+#include "vec/common/hash_table/hash_set.h"
+#include "vec/common/typeid_cast.h"
+#include "vec/data_types/data_type_number.h"
+
+namespace doris::vectorized {
+
+/// uniqExact
+
+template <typename T>
+struct AggregateFunctionUniqExactData {
+ using Key = T;
+
+ /// When creating, the hash table must be small.
+ using Set = HashSet<Key, HashCRC32<Key>, HashTableGrower<4>,
+ HashTableAllocatorWithStackMemory<sizeof(Key) * (1 << 4)>>;
+
+ Set set;
+
+ static String get_name() { return "uniqExact"; }
+};
+
+/// For rows, we put the SipHash values (128 bits) into the hash table.
+template <>
+struct AggregateFunctionUniqExactData<String> {
+ using Key = UInt128;
+
+ /// When creating, the hash table must be small.
+ using Set = HashSet<Key, UInt128TrivialHash, HashTableGrower<3>,
+ HashTableAllocatorWithStackMemory<sizeof(Key) * (1 << 3)>>;
+
+ Set set;
+
+ static String get_name() { return "uniqExact"; }
+};
+
+namespace detail {
+
+/** The structure for the delegation work to add one element to the `uniq` aggregate functions.
+ * Used for partial specialization to add strings.
+ */
+template <typename T, typename Data>
+struct OneAdder {
+ static void ALWAYS_INLINE add(Data& data, const IColumn& column, size_t row_num) {
+ if constexpr (std::is_same_v<T, String>) {
+ StringRef value = column.get_data_at(row_num);
+
+ UInt128 key;
+ SipHash hash;
+ hash.update(value.data, value.size);
+ hash.get128(key.low, key.high);
+
+ data.set.insert(key);
+ } else if constexpr(std::is_same_v<T, Decimal128>) {
+ data.set.insert(assert_cast<const ColumnDecimal<Decimal128>&>(column).get_data()[row_num]);
+ } else {
+ data.set.insert(assert_cast<const ColumnVector<T>&>(column).get_data()[row_num]);
+ }
+ }
+};
+
+} // namespace detail
+
+/// Calculates the number of different values approximately or exactly.
+template <typename T, typename Data>
+class AggregateFunctionUniq final
+ : public IAggregateFunctionDataHelper<Data, AggregateFunctionUniq<T, Data>> {
+public:
+ AggregateFunctionUniq(const DataTypes& argument_types_)
+ : IAggregateFunctionDataHelper<Data, AggregateFunctionUniq<T, Data>>(argument_types_,
+ {}) {}
+
+ String get_name() const override { return Data::get_name(); }
+
+ DataTypePtr get_return_type() const override { return std::make_shared<DataTypeInt64>(); }
+
+ void add(AggregateDataPtr __restrict place, const IColumn** columns, size_t row_num,
+ Arena*) const override {
+ detail::OneAdder<T, Data>::add(this->data(place), *columns[0], row_num);
+ }
+
+ void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena*) const override {
+ this->data(place).set.merge(this->data(rhs).set);
+ }
+
+ void serialize(ConstAggregateDataPtr __restrict place, BufferWritable& buf) const override {
+ this->data(place).set.write(buf);
+ }
+
+ void deserialize(AggregateDataPtr __restrict place, BufferReadable& buf, Arena*) const override {
+ this->data(place).set.read(buf);
+ }
+
+ void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override {
+ assert_cast<ColumnInt64&>(to).get_data().push_back(this->data(place).set.size());
+ }
+
+ const char* get_header_file_path() const override { return __FILE__; }
+};
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/aggregate_functions/aggregate_function_window.cpp b/be/src/vec/aggregate_functions/aggregate_function_window.cpp
new file mode 100644
index 0000000..c40f2bc
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_window.cpp
@@ -0,0 +1,176 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/Processors/Transforms/WindowTransform.cpp
+// and modified by Doris
+
+#include "vec/aggregate_functions/aggregate_function_window.h"
+
+#include "common/logging.h"
+#include "vec/aggregate_functions/aggregate_function_simple_factory.h"
+#include "vec/aggregate_functions/factory_helpers.h"
+#include "vec/aggregate_functions/helpers.h"
+namespace doris::vectorized {
+
+AggregateFunctionPtr create_aggregate_function_dense_rank(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable) {
+ assert_no_parameters(name, parameters);
+
+ return std::make_shared<WindowFunctionDenseRank>(argument_types);
+}
+
+AggregateFunctionPtr create_aggregate_function_rank(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable) {
+ assert_no_parameters(name, parameters);
+
+ return std::make_shared<WindowFunctionRank>(argument_types);
+}
+
+AggregateFunctionPtr create_aggregate_function_row_number(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable) {
+ assert_no_parameters(name, parameters);
+
+ return std::make_shared<WindowFunctionRowNumber>(argument_types);
+}
+
+template <template <typename> class AggregateFunctionTemplate, template <typename> class Data,
+ bool is_nullable, bool is_copy = false>
+static IAggregateFunction* create_function_single_value(const String& name,
+ const DataTypes& argument_types,
+ const Array& parameters) {
+ using StoreType = std::conditional_t<is_copy, CopiedValue, Value>;
+
+ assert_arity_at_most<3>(name, argument_types);
+
+ auto type = argument_types[0].get();
+ if (type->is_nullable()) {
+ type = assert_cast<const DataTypeNullable*>(type)->get_nested_type().get();
+ }
+ WhichDataType which(*type);
+
+#define DISPATCH(TYPE) \
+ if (which.idx == TypeIndex::TYPE) \
+ return new AggregateFunctionTemplate< \
+ Data<LeadAndLagData<TYPE, is_nullable, false, StoreType>>>(argument_types);
+ FOR_NUMERIC_TYPES(DISPATCH)
+#undef DISPATCH
+
+ if (which.is_decimal()) {
+ return new AggregateFunctionTemplate<
+ Data<LeadAndLagData<Int128, is_nullable, false, StoreType>>>(argument_types);
+ }
+ if (which.is_date_or_datetime()) {
+ return new AggregateFunctionTemplate<
+ Data<LeadAndLagData<Int64, is_nullable, false, StoreType>>>(argument_types);
+ }
+ if (which.is_string_or_fixed_string())
+ return new AggregateFunctionTemplate<
+ Data<LeadAndLagData<StringRef, is_nullable, true, StoreType>>>(argument_types);
+ DCHECK(false) << "with unknowed type, failed in create_aggregate_function_leadlag";
+ return nullptr;
+}
+
+template <bool is_nullable>
+AggregateFunctionPtr create_aggregate_function_lag(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable) {
+ return AggregateFunctionPtr(
+ create_function_single_value<WindowFunctionData, WindowFunctionLagData, is_nullable>(
+ name, argument_types, parameters));
+}
+
+template <bool is_nullable>
+AggregateFunctionPtr create_aggregate_function_lead(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable) {
+ return AggregateFunctionPtr(
+ create_function_single_value<WindowFunctionData, WindowFunctionLeadData, is_nullable>(
+ name, argument_types, parameters));
+}
+
+template <bool is_nullable>
+AggregateFunctionPtr create_aggregate_function_first(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable) {
+ return AggregateFunctionPtr(
+ create_function_single_value<WindowFunctionData, WindowFunctionFirstData, is_nullable>(
+ name, argument_types, parameters));
+}
+
+template <bool is_nullable>
+AggregateFunctionPtr create_aggregate_function_last(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable) {
+ return AggregateFunctionPtr(
+ create_function_single_value<WindowFunctionData, WindowFunctionLastData, is_nullable>(
+ name, argument_types, parameters));
+}
+
+AggregateFunctionPtr create_aggregate_function_replace_if_not_null(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable) {
+ return AggregateFunctionPtr(
+ create_function_single_value<WindowFunctionData, WindowFunctionFirstData, false, true>(
+ name, argument_types, parameters));
+}
+
+AggregateFunctionPtr create_aggregate_function_replace(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable) {
+ return AggregateFunctionPtr(
+ create_function_single_value<WindowFunctionData, WindowFunctionFirstData, false, true>(
+ name, argument_types, parameters));
+}
+
+AggregateFunctionPtr create_aggregate_function_replace_nullable(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable) {
+ return AggregateFunctionPtr(
+ create_function_single_value<WindowFunctionData, WindowFunctionFirstData, true, true>(
+ name, argument_types, parameters));
+}
... 62304 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org