You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2020/11/28 10:37:01 UTC
[incubator-doris] branch master updated: [CodeFormat] Clang-format
cpp sources (#4965)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6fedf58 [CodeFormat] Clang-format cpp sources (#4965)
6fedf58 is described below
commit 6fedf5881be01363d3a246e07c79ffb417574d40
Author: sduzh <zh...@qq.com>
AuthorDate: Sat Nov 28 18:36:49 2020 +0800
[CodeFormat] Clang-format cpp sources (#4965)
Clang-format all c++ source files.
---
be/src/agent/agent_server.cpp | 93 +-
be/src/agent/agent_server.h | 5 +-
be/src/agent/cgroups_mgr.cpp | 307 +-
be/src/agent/cgroups_mgr.h | 69 +-
be/src/agent/heartbeat_server.cpp | 63 +-
be/src/agent/heartbeat_server.h | 20 +-
be/src/agent/pusher.cpp | 48 +-
be/src/agent/status.h | 4 +-
be/src/agent/task_worker_pool.cpp | 86 +-
be/src/agent/task_worker_pool.h | 126 +-
be/src/agent/topic_listener.h | 10 +-
be/src/agent/topic_subscriber.cpp | 13 +-
be/src/agent/topic_subscriber.h | 5 +-
be/src/agent/user_resource_listener.cpp | 73 +-
be/src/agent/user_resource_listener.h | 17 +-
be/src/agent/utils.cpp | 70 +-
be/src/agent/utils.h | 33 +-
be/src/common/atomic.h | 44 +-
be/src/common/compiler_util.h | 3 +-
be/src/common/config.h | 1131 ++--
be/src/common/configbase.cpp | 36 +-
be/src/common/configbase.h | 3 +-
be/src/common/daemon.cpp | 117 +-
be/src/common/daemon.h | 2 +-
be/src/common/global_types.h | 2 +-
be/src/common/hdfs.h | 1 -
be/src/common/logconfig.cpp | 35 +-
be/src/common/logging.h | 43 +-
be/src/common/names.h | 6 +-
be/src/common/object_pool.h | 18 +-
be/src/common/resource_tls.cpp | 2 +-
be/src/common/resource_tls.h | 2 +-
be/src/common/status.cpp | 9 +-
be/src/common/status.h | 167 +-
be/src/common/utils.h | 4 +-
be/src/env/env.h | 12 +-
be/src/env/env_posix.cpp | 105 +-
be/src/env/env_util.cpp | 1 -
be/src/exec/aggregation_node.cpp | 99 +-
be/src/exec/aggregation_node.h | 11 +-
be/src/exec/aggregation_node_ir.cpp | 4 +-
be/src/exec/analytic_eval_node.cpp | 159 +-
be/src/exec/analytic_eval_node.h | 7 +-
be/src/exec/base_scanner.cpp | 46 +-
be/src/exec/base_scanner.h | 19 +-
be/src/exec/blocking_join_node.cpp | 23 +-
be/src/exec/blocking_join_node.h | 26 +-
be/src/exec/broker_reader.cpp | 4 +-
be/src/exec/broker_reader.h | 21 +-
be/src/exec/broker_scan_node.cpp | 147 +-
be/src/exec/broker_scan_node.h | 11 +-
be/src/exec/broker_scanner.cpp | 134 +-
be/src/exec/broker_scanner.h | 40 +-
be/src/exec/broker_writer.cpp | 59 +-
be/src/exec/broker_writer.h | 12 +-
be/src/exec/buffered_reader.cpp | 11 +-
be/src/exec/buffered_reader.h | 14 +-
be/src/exec/cross_join_node.cpp | 15 +-
be/src/exec/cross_join_node.h | 8 +-
be/src/exec/csv_scan_node.cpp | 226 +-
be/src/exec/csv_scan_node.h | 29 +-
be/src/exec/csv_scanner.cpp | 105 +-
be/src/exec/csv_scanner.h | 2 +-
be/src/exec/data_sink.cpp | 52 +-
be/src/exec/data_sink.h | 15 +-
be/src/exec/decompressor.cpp | 88 +-
be/src/exec/decompressor.h | 101 +-
be/src/exec/empty_set_node.cpp | 13 +-
be/src/exec/empty_set_node.h | 3 +-
be/src/exec/es/es_predicate.cpp | 221 +-
be/src/exec/es/es_predicate.h | 111 +-
be/src/exec/es/es_query_builder.cpp | 322 +-
be/src/exec/es/es_query_builder.h | 26 +-
be/src/exec/es/es_scan_reader.cpp | 61 +-
be/src/exec/es/es_scan_reader.h | 21 +-
be/src/exec/es/es_scroll_parser.cpp | 439 +-
be/src/exec/es/es_scroll_parser.h | 17 +-
be/src/exec/es/es_scroll_query.cpp | 31 +-
be/src/exec/es/es_scroll_query.h | 18 +-
be/src/exec/es_http_scan_node.cpp | 106 +-
be/src/exec/es_http_scan_node.h | 17 +-
be/src/exec/es_http_scanner.cpp | 13 +-
be/src/exec/es_http_scanner.h | 37 +-
be/src/exec/es_scan_node.cpp | 371 +-
be/src/exec/es_scan_node.h | 16 +-
be/src/exec/except_node.cpp | 6 +-
be/src/exec/exchange_node.cpp | 54 +-
be/src/exec/exchange_node.h | 12 +-
be/src/exec/exec_node.cpp | 180 +-
be/src/exec/exec_node.h | 139 +-
be/src/exec/file_reader.h | 7 +-
be/src/exec/file_writer.h | 3 +-
be/src/exec/hash_join_node.cpp | 134 +-
be/src/exec/hash_join_node.h | 34 +-
be/src/exec/hash_table.cpp | 49 +-
be/src/exec/hash_table.h | 100 +-
be/src/exec/intersect_node.cpp | 6 +-
be/src/exec/json_scanner.cpp | 258 +-
be/src/exec/json_scanner.h | 81 +-
be/src/exec/line_reader.h | 5 +-
be/src/exec/local_file_reader.cpp | 31 +-
be/src/exec/local_file_reader.h | 8 +-
be/src/exec/local_file_writer.cpp | 20 +-
be/src/exec/lzo_decompressor.cpp | 91 +-
be/src/exec/merge_join_node.cpp | 63 +-
be/src/exec/merge_join_node.h | 8 +-
be/src/exec/merge_node.cpp | 55 +-
be/src/exec/merge_node.h | 7 +-
be/src/exec/mysql_scan_node.cpp | 43 +-
be/src/exec/mysql_scan_node.h | 10 +-
be/src/exec/mysql_scanner.cpp | 30 +-
be/src/exec/mysql_scanner.h | 17 +-
be/src/exec/odbc_scan_node.cpp | 31 +-
be/src/exec/odbc_scan_node.h | 11 +-
be/src/exec/odbc_scanner.cpp | 92 +-
be/src/exec/odbc_scanner.h | 32 +-
be/src/exec/olap_common.cpp | 17 +-
be/src/exec/olap_common.h | 247 +-
be/src/exec/olap_rewrite_node.cpp | 58 +-
be/src/exec/olap_rewrite_node.h | 4 +-
be/src/exec/olap_scan_node.cpp | 361 +-
be/src/exec/olap_scan_node.h | 35 +-
be/src/exec/olap_scanner.cpp | 141 +-
be/src/exec/olap_scanner.h | 56 +-
be/src/exec/olap_utils.h | 52 +-
be/src/exec/orc_scanner.cpp | 361 +-
be/src/exec/orc_scanner.h | 6 +-
be/src/exec/parquet_reader.cpp | 582 ++-
be/src/exec/parquet_reader.h | 41 +-
be/src/exec/parquet_scanner.cpp | 96 +-
be/src/exec/parquet_scanner.h | 23 +-
be/src/exec/parquet_writer.cpp | 19 +-
be/src/exec/parquet_writer.h | 25 +-
be/src/exec/partitioned_aggregation_node.cc | 2439 +++++----
be/src/exec/partitioned_aggregation_node.h | 1086 ++--
be/src/exec/partitioned_aggregation_node_ir.cc | 381 +-
be/src/exec/partitioned_hash_table.cc | 806 ++-
be/src/exec/partitioned_hash_table.h | 1657 +++---
be/src/exec/partitioned_hash_table.inline.h | 572 +-
be/src/exec/partitioned_hash_table_ir.cc | 9 +-
be/src/exec/pl_task_root.cpp | 23 +-
be/src/exec/pl_task_root.h | 9 +-
be/src/exec/plain_text_line_reader.cpp | 97 +-
be/src/exec/plain_text_line_reader.h | 19 +-
be/src/exec/read_write_util.cpp | 2 +-
be/src/exec/read_write_util.h | 25 +-
be/src/exec/repeat_node.cpp | 72 +-
be/src/exec/repeat_node.h | 2 +-
be/src/exec/row_batch_list.h | 30 +-
be/src/exec/scan_node.cpp | 14 +-
be/src/exec/scan_node.h | 25 +-
be/src/exec/scanner_ir.cpp | 27 +-
be/src/exec/schema_scan_node.cpp | 53 +-
be/src/exec/schema_scan_node.h | 8 +-
be/src/exec/schema_scanner.cpp | 61 +-
be/src/exec/schema_scanner.h | 32 +-
.../schema_scanner/schema_charsets_scanner.cpp | 52 +-
.../exec/schema_scanner/schema_charsets_scanner.h | 13 +-
.../schema_scanner/schema_collations_scanner.cpp | 65 +-
.../schema_scanner/schema_collations_scanner.h | 15 +-
.../exec/schema_scanner/schema_columns_scanner.cpp | 352 +-
.../exec/schema_scanner/schema_columns_scanner.h | 17 +-
.../exec/schema_scanner/schema_dummy_scanner.cpp | 20 +-
be/src/exec/schema_scanner/schema_dummy_scanner.h | 8 +-
be/src/exec/schema_scanner/schema_helper.cpp | 129 +-
be/src/exec/schema_scanner/schema_helper.h | 66 +-
.../schema_schema_privileges_scanner.cpp | 55 +-
.../schema_schema_privileges_scanner.h | 10 +-
.../schema_scanner/schema_schemata_scanner.cpp | 59 +-
.../exec/schema_scanner/schema_schemata_scanner.h | 8 +-
.../schema_table_privileges_scanner.cpp | 58 +-
.../schema_table_privileges_scanner.h | 10 +-
.../exec/schema_scanner/schema_tables_scanner.cpp | 161 +-
be/src/exec/schema_scanner/schema_tables_scanner.h | 8 +-
.../schema_user_privileges_scanner.cpp | 50 +-
.../schema_user_privileges_scanner.h | 10 +-
.../schema_scanner/schema_variables_scanner.cpp | 49 +-
.../exec/schema_scanner/schema_variables_scanner.h | 15 +-
.../exec/schema_scanner/schema_views_scanner.cpp | 94 +-
be/src/exec/schema_scanner/schema_views_scanner.h | 8 +-
be/src/exec/select_node.cpp | 23 +-
be/src/exec/select_node.h | 2 +-
be/src/exec/set_operation_node.h | 1 -
be/src/exec/sort_exec_exprs.cpp | 31 +-
be/src/exec/sort_exec_exprs.h | 4 +-
be/src/exec/spill_sort_node.cc | 32 +-
be/src/exec/spill_sort_node.h | 2 +-
be/src/exec/tablet_info.cpp | 70 +-
be/src/exec/tablet_info.h | 58 +-
be/src/exec/tablet_sink.cpp | 19 +-
be/src/exec/tablet_sink.h | 4 +-
be/src/exec/text_converter.cpp | 9 +-
be/src/exec/text_converter.h | 6 +-
be/src/exec/topn_node.cpp | 60 +-
be/src/exec/topn_node.h | 11 +-
be/src/exec/union_node.cpp | 54 +-
be/src/exec/union_node.h | 17 +-
be/src/exec/union_node_ir.cpp | 11 +-
be/src/exprs/agg_fn.cc | 265 +-
be/src/exprs/agg_fn.h | 201 +-
be/src/exprs/agg_fn_evaluator.cpp | 375 +-
be/src/exprs/agg_fn_evaluator.h | 137 +-
be/src/exprs/aggregate_functions.cpp | 1351 +++--
be/src/exprs/aggregate_functions.h | 246 +-
be/src/exprs/anyval_util.cpp | 23 +-
be/src/exprs/anyval_util.h | 128 +-
be/src/exprs/arithmetic_expr.cpp | 121 +-
be/src/exprs/arithmetic_expr.h | 79 +-
be/src/exprs/binary_predicate.cpp | 197 +-
be/src/exprs/binary_predicate.h | 50 +-
be/src/exprs/bitmap_function.cpp | 366 +-
be/src/exprs/bitmap_function.h | 30 +-
be/src/exprs/case_expr.cpp | 118 +-
be/src/exprs/case_expr.h | 31 +-
be/src/exprs/cast_expr.cpp | 20 +-
be/src/exprs/cast_expr.h | 34 +-
be/src/exprs/cast_functions.cpp | 230 +-
be/src/exprs/cast_functions.h | 2 +-
be/src/exprs/compound_predicate.cpp | 11 +-
be/src/exprs/compound_predicate.h | 26 +-
be/src/exprs/conditional_functions.cpp | 14 +-
be/src/exprs/conditional_functions.h | 25 +-
be/src/exprs/conditional_functions_ir.cpp | 67 +-
be/src/exprs/decimal_operators.cpp | 136 +-
be/src/exprs/decimal_operators.h | 49 +-
be/src/exprs/decimalv2_operators.cpp | 146 +-
be/src/exprs/decimalv2_operators.h | 49 +-
be/src/exprs/encryption_functions.cpp | 47 +-
be/src/exprs/encryption_functions.h | 21 +-
be/src/exprs/es_functions.cpp | 13 +-
be/src/exprs/es_functions.h | 9 +-
be/src/exprs/expr.cpp | 365 +-
be/src/exprs/expr.h | 127 +-
be/src/exprs/expr_context.cpp | 137 +-
be/src/exprs/expr_context.h | 24 +-
be/src/exprs/expr_ir.cpp | 8 +-
be/src/exprs/expr_value.h | 58 +-
be/src/exprs/grouping_sets_functions.cpp | 14 +-
be/src/exprs/grouping_sets_functions.h | 11 +-
be/src/exprs/hash_functions.cpp | 8 +-
be/src/exprs/hash_functions.h | 9 +-
be/src/exprs/hll_function.cpp | 15 +-
be/src/exprs/hll_function.h | 4 +-
be/src/exprs/hll_hash_function.cpp | 8 +-
be/src/exprs/hll_hash_function.h | 4 +-
be/src/exprs/hybrid_map.h | 17 +-
be/src/exprs/hybrid_set.cpp | 26 +-
be/src/exprs/hybrid_set.h | 94 +-
be/src/exprs/in_predicate.cpp | 31 +-
be/src/exprs/in_predicate.h | 28 +-
be/src/exprs/info_func.cpp | 13 +-
be/src/exprs/info_func.h | 12 +-
be/src/exprs/is_null_predicate.cpp | 10 +-
be/src/exprs/is_null_predicate.h | 7 +-
be/src/exprs/json_functions.cpp | 127 +-
be/src/exprs/json_functions.h | 91 +-
be/src/exprs/like_predicate.cpp | 196 +-
be/src/exprs/like_predicate.h | 137 +-
be/src/exprs/literal.cpp | 25 +-
be/src/exprs/literal.h | 6 +-
be/src/exprs/math_functions.cpp | 356 +-
be/src/exprs/math_functions.h | 255 +-
be/src/exprs/new_agg_fn_evaluator.cc | 853 ++-
be/src/exprs/new_agg_fn_evaluator.h | 481 +-
be/src/exprs/new_agg_fn_evaluator_ir.cc | 4 +-
be/src/exprs/new_in_predicate.cpp | 133 +-
be/src/exprs/new_in_predicate.h | 403 +-
be/src/exprs/null_literal.cpp | 6 +-
be/src/exprs/null_literal.h | 5 +-
be/src/exprs/operators.cpp | 153 +-
be/src/exprs/operators.h | 410 +-
be/src/exprs/predicate.h | 4 +-
be/src/exprs/scalar_fn_call.cpp | 264 +-
be/src/exprs/scalar_fn_call.h | 20 +-
be/src/exprs/slot_ref.cpp | 59 +-
be/src/exprs/slot_ref.h | 36 +-
be/src/exprs/string_functions.cpp | 205 +-
be/src/exprs/string_functions.h | 219 +-
be/src/exprs/time_operators.cpp | 52 +-
be/src/exprs/time_operators.h | 3 +-
be/src/exprs/timestamp_functions.cpp | 528 +-
be/src/exprs/timestamp_functions.h | 646 +--
be/src/exprs/tuple_is_null_predicate.cpp | 15 +-
be/src/exprs/tuple_is_null_predicate.h | 9 +-
be/src/exprs/udf_builtins.cpp | 20 +-
be/src/exprs/udf_builtins.h | 34 +-
be/src/geo/geo_common.cpp | 2 +-
be/src/geo/geo_common.h | 2 +-
be/src/geo/geo_functions.cpp | 49 +-
be/src/geo/geo_functions.h | 53 +-
be/src/geo/geo_types.cpp | 19 +-
be/src/geo/geo_types.h | 32 +-
be/src/geo/wkt_parse.cpp | 4 +-
be/src/geo/wkt_parse.h | 2 +-
be/src/geo/wkt_parse_ctx.h | 1 -
be/src/geo/wkt_parse_type.h | 13 +-
be/src/gutil/arm_instruction_set_select.h | 46 +-
be/src/gutil/atomic_refcount.h | 96 +-
be/src/gutil/atomicops-internals-gcc.h | 161 +-
be/src/gutil/atomicops-internals-tsan.h | 211 +-
be/src/gutil/atomicops-internals-x86.cc | 101 +-
be/src/gutil/atomicops-internals-x86.h | 508 +-
be/src/gutil/atomicops.h | 170 +-
be/src/gutil/basictypes.h | 13 +-
be/src/gutil/bits.cc | 126 +-
be/src/gutil/bits.h | 336 +-
be/src/gutil/casts.h | 253 +-
be/src/gutil/charmap.h | 90 +-
be/src/gutil/cpu.cc | 421 +-
be/src/gutil/cpu.h | 141 +-
be/src/gutil/cycleclock-inl.h | 167 +-
be/src/gutil/dynamic_annotations.h | 713 ++-
be/src/gutil/endian.h | 483 +-
be/src/gutil/gscoped_ptr.h | 1158 ++---
be/src/gutil/hash/builtin_type_hash.h | 80 +-
be/src/gutil/hash/city.cc | 433 +-
be/src/gutil/hash/city.h | 17 +-
be/src/gutil/hash/hash.cc | 277 +-
be/src/gutil/hash/hash.h | 315 +-
be/src/gutil/hash/hash128to64.h | 18 +-
be/src/gutil/hash/jenkins.cc | 367 +-
be/src/gutil/hash/jenkins.h | 9 +-
be/src/gutil/hash/jenkins_lookup2.h | 143 +-
be/src/gutil/hash/legacy_hash.h | 41 +-
be/src/gutil/hash/string_hash.h | 63 +-
be/src/gutil/int128.cc | 8 +-
be/src/gutil/int128.h | 389 +-
be/src/gutil/integral_types.h | 59 +-
be/src/gutil/linux_syscall_support.h | 3810 +++++++-------
be/src/gutil/logging-inl.h | 2 +-
be/src/gutil/macros.h | 68 +-
be/src/gutil/map-util.h | 778 ++-
be/src/gutil/move.h | 30 +-
be/src/gutil/once.cc | 74 +-
be/src/gutil/once.h | 94 +-
be/src/gutil/port.h | 619 ++-
be/src/gutil/ref_counted.cc | 69 +-
be/src/gutil/ref_counted.h | 359 +-
be/src/gutil/spinlock_internal.cc | 101 +-
be/src/gutil/spinlock_internal.h | 15 +-
be/src/gutil/spinlock_linux-inl.h | 75 +-
be/src/gutil/spinlock_posix-inl.h | 31 +-
be/src/gutil/spinlock_win32-inl.h | 18 +-
be/src/gutil/stl_util.h | 903 ++--
be/src/gutil/stringprintf.cc | 186 +-
be/src/gutil/stringprintf.h | 13 +-
be/src/gutil/strings/ascii_ctype.cc | 116 +-
be/src/gutil/strings/ascii_ctype.h | 50 +-
be/src/gutil/strings/charset.cc | 14 +-
be/src/gutil/strings/charset.h | 62 +-
be/src/gutil/strings/escaping.cc | 2768 +++++-----
be/src/gutil/strings/escaping.h | 150 +-
be/src/gutil/strings/fastmem.h | 190 +-
be/src/gutil/strings/join.cc | 216 +-
be/src/gutil/strings/join.h | 300 +-
be/src/gutil/strings/memutil.cc | 200 +-
be/src/gutil/strings/memutil.h | 119 +-
be/src/gutil/strings/numbers.cc | 1981 ++++---
be/src/gutil/strings/numbers.h | 178 +-
be/src/gutil/strings/split.cc | 1268 +++--
be/src/gutil/strings/split.h | 360 +-
be/src/gutil/strings/split_internal.h | 504 +-
be/src/gutil/strings/strcat.cc | 395 +-
be/src/gutil/strings/strcat.h | 434 +-
be/src/gutil/strings/stringpiece.cc | 232 +-
be/src/gutil/strings/stringpiece.h | 392 +-
be/src/gutil/strings/strip.cc | 439 +-
be/src/gutil/strings/strip.h | 70 +-
be/src/gutil/strings/substitute.cc | 188 +-
be/src/gutil/strings/substitute.h | 215 +-
be/src/gutil/strings/util.cc | 1607 +++---
be/src/gutil/strings/util.h | 212 +-
be/src/gutil/strtoint.cc | 64 +-
be/src/gutil/strtoint.h | 58 +-
be/src/gutil/sysinfo-test.cc | 82 +-
be/src/gutil/sysinfo.cc | 639 ++-
be/src/gutil/sysinfo.h | 8 +-
be/src/gutil/template_util.h | 90 +-
be/src/gutil/threading/thread_collision_warner.cc | 69 +-
be/src/gutil/threading/thread_collision_warner.h | 199 +-
be/src/gutil/type_traits.h | 427 +-
be/src/gutil/utf/utf.h | 50 +-
be/src/gutil/utf/utfdef.h | 10 +-
be/src/gutil/valgrind.h | 5468 ++++++++++----------
be/src/gutil/walltime.h | 165 +-
be/src/http/action/checksum_action.cpp | 40 +-
be/src/http/action/checksum_action.h | 12 +-
be/src/http/action/compaction_action.cpp | 65 +-
be/src/http/action/compaction_action.h | 15 +-
be/src/http/action/health_action.cpp | 9 +-
be/src/http/action/health_action.h | 9 +-
be/src/http/action/meta_action.cpp | 5 +-
be/src/http/action/meta_action.h | 12 +-
be/src/http/action/metrics_action.cpp | 7 +-
be/src/http/action/metrics_action.h | 9 +-
be/src/http/action/mini_load.cpp | 313 +-
be/src/http/action/mini_load.h | 46 +-
be/src/http/action/pprof_actions.cpp | 17 +-
be/src/http/action/pprof_actions.h | 2 +-
be/src/http/action/reload_tablet_action.cpp | 34 +-
be/src/http/action/reload_tablet_action.h | 15 +-
be/src/http/action/restore_tablet_action.h | 27 +-
be/src/http/action/snapshot_action.cpp | 30 +-
be/src/http/action/snapshot_action.h | 9 +-
be/src/http/action/stream_load.cpp | 88 +-
be/src/http/action/stream_load.h | 4 +-
be/src/http/action/tablets_info_action.cpp | 7 +-
be/src/http/action/tablets_info_action.h | 6 +-
be/src/http/action/update_config_action.cpp | 18 +-
be/src/http/default_path_handlers.cpp | 118 +-
be/src/http/default_path_handlers.h | 1 +
be/src/http/download_action.cpp | 29 +-
be/src/http/download_action.h | 15 +-
be/src/http/ev_http_server.cpp | 86 +-
be/src/http/ev_http_server.h | 13 +-
be/src/http/http_channel.cpp | 28 +-
be/src/http/http_channel.h | 7 +-
be/src/http/http_client.cpp | 39 +-
be/src/http/http_client.h | 22 +-
be/src/http/http_common.h | 6 +-
be/src/http/http_handler.h | 10 +-
be/src/http/http_headers.cpp | 2 +-
be/src/http/http_headers.h | 2 +-
be/src/http/http_method.cpp | 24 +-
be/src/http/http_method.h | 16 +-
be/src/http/http_parser.cpp | 21 +-
be/src/http/http_parser.h | 22 +-
be/src/http/http_request.cpp | 39 +-
be/src/http/http_request.h | 39 +-
be/src/http/http_response.cpp | 27 +-
be/src/http/http_response.h | 18 +-
be/src/http/http_status.cpp | 83 +-
be/src/http/http_status.h | 2 +-
be/src/http/monitor_action.cpp | 19 +-
be/src/http/monitor_action.h | 12 +-
be/src/http/rest_monitor_iface.h | 4 +-
be/src/http/utils.cpp | 13 +-
be/src/http/utils.h | 6 +-
be/src/http/web_page_handler.cpp | 9 +-
be/src/http/web_page_handler.h | 19 +-
be/src/olap/aggregate_func.cpp | 17 +-
be/src/olap/aggregate_func.h | 138 +-
be/src/olap/base_compaction.cpp | 21 +-
be/src/olap/base_compaction.h | 10 +-
be/src/olap/base_tablet.cpp | 5 +-
be/src/olap/base_tablet.h | 1 +
be/src/olap/bhp_lib.h | 1261 +++--
be/src/olap/bloom_filter_reader.cpp | 15 +-
be/src/olap/bloom_filter_reader.h | 8 +-
be/src/olap/bloom_filter_writer.cpp | 8 +-
be/src/olap/byte_buffer.cpp | 44 +-
be/src/olap/byte_buffer.h | 52 +-
be/src/olap/collection.h | 19 +-
be/src/olap/column_block.h | 23 +-
be/src/olap/column_mapping.h | 2 +-
be/src/olap/column_predicate.h | 9 +-
be/src/olap/column_vector.cpp | 194 +-
be/src/olap/column_vector.h | 73 +-
be/src/olap/compaction.cpp | 72 +-
be/src/olap/compaction.h | 13 +-
be/src/olap/comparison_predicate.cpp | 444 +-
be/src/olap/comparison_predicate.h | 26 +-
be/src/olap/compress.cpp | 38 +-
be/src/olap/compress.h | 6 +-
be/src/olap/cumulative_compaction.cpp | 23 +-
be/src/olap/cumulative_compaction.h | 13 +-
be/src/olap/cumulative_compaction_policy.cpp | 128 +-
be/src/olap/cumulative_compaction_policy.h | 73 +-
be/src/olap/data_dir.cpp | 131 +-
be/src/olap/data_dir.h | 4 +-
be/src/olap/decimal12.h | 42 +-
be/src/olap/delete_handler.cpp | 53 +-
be/src/olap/delete_handler.h | 30 +-
be/src/olap/delta_writer.cpp | 25 +-
be/src/olap/delta_writer.h | 10 +-
be/src/olap/field.h | 227 +-
be/src/olap/file_helper.cpp | 93 +-
be/src/olap/file_helper.h | 155 +-
be/src/olap/file_stream.cpp | 70 +-
be/src/olap/file_stream.h | 80 +-
be/src/olap/fs/block_id.h | 37 +-
be/src/olap/fs/block_manager.h | 7 +-
be/src/olap/fs/block_manager_metrics.h | 1 -
be/src/olap/fs/file_block_manager.cpp | 109 +-
be/src/olap/fs/file_block_manager.h | 1 -
be/src/olap/fs/fs_util.cpp | 2 +-
be/src/olap/generic_iterators.cpp | 42 +-
be/src/olap/generic_iterators.h | 2 +-
be/src/olap/hll.cpp | 176 +-
be/src/olap/hll.h | 89 +-
be/src/olap/in_list_predicate.cpp | 332 +-
be/src/olap/in_list_predicate.h | 32 +-
be/src/olap/in_stream.cpp | 66 +-
be/src/olap/in_stream.h | 28 +-
be/src/olap/iterators.h | 30 +-
be/src/olap/key_coder.cpp | 9 +-
be/src/olap/key_coder.h | 104 +-
be/src/olap/lru_cache.cpp | 77 +-
be/src/olap/lru_cache.h | 728 ++-
be/src/olap/memtable.cpp | 14 +-
be/src/olap/memtable.h | 5 +-
be/src/olap/memtable_flush_executor.h | 11 +-
be/src/olap/merger.cpp | 35 +-
be/src/olap/merger.h | 10 +-
be/src/olap/null_predicate.cpp | 12 +-
be/src/olap/null_predicate.h | 5 +-
be/src/olap/olap_common.h | 109 +-
be/src/olap/olap_cond.cpp | 69 +-
be/src/olap/olap_cond.h | 64 +-
be/src/olap/olap_define.h | 98 +-
be/src/olap/olap_index.cpp | 64 +-
be/src/olap/olap_index.h | 134 +-
be/src/olap/olap_meta.cpp | 24 +-
be/src/olap/olap_meta.h | 8 +-
be/src/olap/olap_server.cpp | 100 +-
be/src/olap/olap_snapshot_converter.cpp | 89 +-
be/src/olap/olap_snapshot_converter.h | 35 +-
be/src/olap/options.cpp | 9 +-
be/src/olap/options.h | 13 +-
be/src/olap/out_stream.h | 64 +-
be/src/olap/page_cache.cpp | 12 +-
be/src/olap/page_cache.h | 14 +-
be/src/olap/push_handler.cpp | 999 ++--
be/src/olap/push_handler.h | 113 +-
be/src/olap/reader.cpp | 473 +-
be/src/olap/reader.h | 47 +-
be/src/olap/row.h | 67 +-
be/src/olap/row_block.cpp | 19 +-
be/src/olap/row_block.h | 24 +-
be/src/olap/row_block2.cpp | 19 +-
be/src/olap/row_block2.h | 33 +-
be/src/olap/row_cursor.cpp | 48 +-
be/src/olap/row_cursor.h | 61 +-
be/src/olap/row_cursor_cell.h | 7 +-
be/src/olap/rowset/alpha_rowset.cpp | 122 +-
be/src/olap/rowset/alpha_rowset.h | 6 +-
be/src/olap/rowset/alpha_rowset_meta.cpp | 2 +-
be/src/olap/rowset/alpha_rowset_meta.h | 9 +-
be/src/olap/rowset/alpha_rowset_reader.cpp | 53 +-
be/src/olap/rowset/alpha_rowset_reader.h | 18 +-
be/src/olap/rowset/alpha_rowset_writer.cpp | 94 +-
be/src/olap/rowset/alpha_rowset_writer.h | 28 +-
be/src/olap/rowset/beta_rowset.cpp | 32 +-
be/src/olap/rowset/beta_rowset_reader.cpp | 15 +-
be/src/olap/rowset/beta_rowset_reader.h | 4 +-
be/src/olap/rowset/beta_rowset_writer.cpp | 53 +-
be/src/olap/rowset/beta_rowset_writer.h | 17 +-
be/src/olap/rowset/bit_field_reader.cpp | 13 +-
be/src/olap/rowset/bit_field_reader.h | 4 +-
be/src/olap/rowset/bit_field_writer.cpp | 15 +-
be/src/olap/rowset/bit_field_writer.h | 5 +-
be/src/olap/rowset/column_data.cpp | 122 +-
be/src/olap/rowset/column_data.h | 86 +-
be/src/olap/rowset/column_data_writer.cpp | 80 +-
be/src/olap/rowset/column_data_writer.h | 22 +-
be/src/olap/rowset/column_reader.cpp | 394 +-
be/src/olap/rowset/column_reader.h | 559 +-
be/src/olap/rowset/column_writer.cpp | 311 +-
be/src/olap/rowset/column_writer.h | 192 +-
be/src/olap/rowset/rowset.cpp | 14 +-
be/src/olap/rowset/rowset.h | 91 +-
be/src/olap/rowset/rowset_converter.cpp | 7 +-
be/src/olap/rowset/rowset_converter.h | 15 +-
be/src/olap/rowset/rowset_factory.cpp | 14 +-
be/src/olap/rowset/rowset_factory.h | 7 +-
be/src/olap/rowset/rowset_id_generator.h | 4 +-
be/src/olap/rowset/rowset_meta.h | 168 +-
be/src/olap/rowset/rowset_meta_manager.cpp | 58 +-
be/src/olap/rowset/rowset_meta_manager.h | 17 +-
be/src/olap/rowset/rowset_reader.h | 8 +-
be/src/olap/rowset/rowset_reader_context.h | 2 +-
be/src/olap/rowset/rowset_writer.h | 8 +-
be/src/olap/rowset/rowset_writer_context.h | 26 +-
be/src/olap/rowset/run_length_byte_reader.cpp | 11 +-
be/src/olap/rowset/run_length_byte_reader.h | 4 +-
be/src/olap/rowset/run_length_byte_writer.cpp | 9 +-
be/src/olap/rowset/run_length_byte_writer.h | 5 +-
be/src/olap/rowset/run_length_integer_reader.cpp | 10 +-
be/src/olap/rowset/run_length_integer_reader.h | 8 +-
be/src/olap/rowset/run_length_integer_writer.cpp | 16 +-
be/src/olap/rowset/run_length_integer_writer.h | 52 +-
be/src/olap/rowset/segment_group.cpp | 360 +-
be/src/olap/rowset/segment_group.h | 140 +-
be/src/olap/rowset/segment_reader.cpp | 194 +-
be/src/olap/rowset/segment_reader.h | 117 +-
be/src/olap/rowset/segment_v2/binary_dict_page.cpp | 62 +-
be/src/olap/rowset/segment_v2/binary_dict_page.h | 28 +-
be/src/olap/rowset/segment_v2/binary_plain_page.h | 61 +-
.../olap/rowset/segment_v2/binary_prefix_page.cpp | 28 +-
be/src/olap/rowset/segment_v2/binary_prefix_page.h | 36 +-
.../olap/rowset/segment_v2/bitmap_index_reader.cpp | 3 +-
.../olap/rowset/segment_v2/bitmap_index_reader.h | 24 +-
.../olap/rowset/segment_v2/bitmap_index_writer.cpp | 89 +-
.../olap/rowset/segment_v2/bitmap_index_writer.h | 3 +-
be/src/olap/rowset/segment_v2/bitshuffle_page.cpp | 27 +-
be/src/olap/rowset/segment_v2/bitshuffle_page.h | 136 +-
.../olap/rowset/segment_v2/bitshuffle_wrapper.cpp | 9 +-
.../rowset/segment_v2/block_split_bloom_filter.cpp | 3 +-
.../rowset/segment_v2/block_split_bloom_filter.h | 2 +-
be/src/olap/rowset/segment_v2/bloom_filter.cpp | 7 +-
be/src/olap/rowset/segment_v2/bloom_filter.h | 26 +-
.../segment_v2/bloom_filter_index_reader.cpp | 11 +-
.../rowset/segment_v2/bloom_filter_index_reader.h | 28 +-
.../segment_v2/bloom_filter_index_writer.cpp | 86 +-
.../rowset/segment_v2/bloom_filter_index_writer.h | 7 +-
be/src/olap/rowset/segment_v2/column_reader.cpp | 192 +-
be/src/olap/rowset/segment_v2/column_reader.h | 74 +-
be/src/olap/rowset/segment_v2/column_writer.cpp | 146 +-
be/src/olap/rowset/segment_v2/column_writer.h | 48 +-
.../rowset/segment_v2/empty_segment_iterator.cpp | 2 +-
.../rowset/segment_v2/empty_segment_iterator.h | 2 +-
be/src/olap/rowset/segment_v2/encoding_info.cpp | 103 +-
be/src/olap/rowset/segment_v2/encoding_info.h | 16 +-
.../rowset/segment_v2/frame_of_reference_page.h | 52 +-
be/src/olap/rowset/segment_v2/index_page.cpp | 6 +-
be/src/olap/rowset/segment_v2/index_page.h | 21 +-
.../rowset/segment_v2/indexed_column_reader.cpp | 23 +-
.../olap/rowset/segment_v2/indexed_column_reader.h | 21 +-
.../rowset/segment_v2/indexed_column_writer.cpp | 25 +-
.../olap/rowset/segment_v2/indexed_column_writer.h | 5 +-
be/src/olap/rowset/segment_v2/options.h | 5 +-
.../olap/rowset/segment_v2/ordinal_page_index.cpp | 10 +-
be/src/olap/rowset/segment_v2/ordinal_page_index.h | 24 +-
be/src/olap/rowset/segment_v2/page_builder.h | 6 +-
be/src/olap/rowset/segment_v2/page_decoder.h | 8 +-
be/src/olap/rowset/segment_v2/page_handle.h | 18 +-
be/src/olap/rowset/segment_v2/page_io.cpp | 27 +-
be/src/olap/rowset/segment_v2/page_io.h | 29 +-
be/src/olap/rowset/segment_v2/page_pointer.h | 22 +-
be/src/olap/rowset/segment_v2/parsed_page.h | 27 +-
be/src/olap/rowset/segment_v2/plain_page.h | 68 +-
be/src/olap/rowset/segment_v2/rle_page.h | 120 +-
be/src/olap/rowset/segment_v2/row_ranges.h | 68 +-
be/src/olap/rowset/segment_v2/segment.cpp | 60 +-
be/src/olap/rowset/segment_v2/segment.h | 21 +-
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 112 +-
be/src/olap/rowset/segment_v2/segment_iterator.h | 21 +-
be/src/olap/rowset/segment_v2/segment_writer.cpp | 32 +-
be/src/olap/rowset/segment_v2/segment_writer.h | 12 +-
be/src/olap/rowset/segment_v2/zone_map_index.cpp | 9 +-
be/src/olap/rowset/segment_v2/zone_map_index.h | 14 +-
be/src/olap/rowset/segment_writer.cpp | 103 +-
be/src/olap/rowset/segment_writer.h | 14 +-
be/src/olap/schema.cpp | 10 +-
be/src/olap/schema.h | 29 +-
be/src/olap/schema_change.cpp | 288 +-
be/src/olap/schema_change.h | 29 +-
be/src/olap/selection_vector.h | 25 +-
be/src/olap/serialize.cpp | 110 +-
be/src/olap/serialize.h | 56 +-
be/src/olap/short_key_index.cpp | 17 +-
be/src/olap/short_key_index.h | 34 +-
be/src/olap/skiplist.h | 127 +-
be/src/olap/snapshot_manager.cpp | 162 +-
be/src/olap/snapshot_manager.h | 71 +-
be/src/olap/storage_engine.cpp | 201 +-
be/src/olap/storage_engine.h | 66 +-
be/src/olap/stream_index_common.cpp | 16 +-
be/src/olap/stream_index_common.h | 33 +-
be/src/olap/stream_index_reader.cpp | 39 +-
be/src/olap/stream_index_reader.h | 24 +-
be/src/olap/stream_index_writer.cpp | 14 +-
be/src/olap/stream_index_writer.h | 3 +-
be/src/olap/stream_name.cpp | 17 +-
be/src/olap/stream_name.h | 17 +-
be/src/olap/tablet.cpp | 218 +-
be/src/olap/tablet.h | 46 +-
be/src/olap/tablet_manager.cpp | 300 +-
be/src/olap/tablet_manager.h | 47 +-
be/src/olap/tablet_meta.cpp | 165 +-
be/src/olap/tablet_meta.h | 22 +-
be/src/olap/tablet_meta_manager.cpp | 59 +-
be/src/olap/tablet_meta_manager.h | 31 +-
be/src/olap/tablet_schema.cpp | 212 +-
be/src/olap/tablet_schema.h | 13 +-
be/src/olap/tablet_sync_service.cpp | 48 +-
be/src/olap/tablet_sync_service.h | 36 +-
be/src/olap/task/engine_alter_tablet_task.cpp | 23 +-
be/src/olap/task/engine_alter_tablet_task.h | 6 +-
be/src/olap/task/engine_batch_load_task.cpp | 122 +-
be/src/olap/task/engine_batch_load_task.h | 17 +-
be/src/olap/task/engine_checksum_task.cpp | 37 +-
be/src/olap/task/engine_checksum_task.h | 11 +-
be/src/olap/task/engine_clone_task.cpp | 417 +-
be/src/olap/task/engine_clone_task.h | 62 +-
be/src/olap/task/engine_publish_version_task.cpp | 44 +-
be/src/olap/task/engine_publish_version_task.h | 3 +-
be/src/olap/task/engine_storage_migration_task.cpp | 54 +-
be/src/olap/task/engine_storage_migration_task.h | 7 +-
be/src/olap/tuple.h | 16 +-
be/src/olap/txn_manager.cpp | 240 +-
be/src/olap/txn_manager.h | 103 +-
be/src/olap/types.cpp | 72 +-
be/src/olap/types.h | 299 +-
be/src/olap/uint24.h | 32 +-
be/src/olap/utils.cpp | 1167 ++---
be/src/olap/utils.h | 139 +-
be/src/olap/version_graph.cpp | 98 +-
be/src/olap/version_graph.h | 55 +-
be/src/olap/wrapper_field.cpp | 29 +-
be/src/olap/wrapper_field.h | 32 +-
be/src/plugin/plugin.h | 32 +-
be/src/plugin/plugin_loader.cpp | 55 +-
be/src/plugin/plugin_loader.h | 55 +-
be/src/plugin/plugin_mgr.cpp | 53 +-
be/src/plugin/plugin_mgr.h | 31 +-
be/src/plugin/plugin_zip.cpp | 47 +-
be/src/plugin/plugin_zip.h | 4 +-
be/src/runtime/broker_mgr.cpp | 24 +-
be/src/runtime/broker_mgr.h | 7 +-
be/src/runtime/buffer_control_block.cpp | 26 +-
be/src/runtime/buffer_control_block.h | 32 +-
be/src/runtime/buffered_block_mgr2.cc | 311 +-
be/src/runtime/buffered_block_mgr2.h | 58 +-
be/src/runtime/buffered_tuple_stream2.cc | 151 +-
be/src/runtime/buffered_tuple_stream2.h | 36 +-
be/src/runtime/buffered_tuple_stream2.inline.h | 9 +-
be/src/runtime/buffered_tuple_stream2_ir.cc | 10 +-
be/src/runtime/buffered_tuple_stream3.cc | 1529 +++---
be/src/runtime/buffered_tuple_stream3.h | 985 ++--
be/src/runtime/buffered_tuple_stream3.inline.h | 37 +-
be/src/runtime/bufferpool/buffer_allocator.cc | 1108 ++--
be/src/runtime/bufferpool/buffer_allocator.h | 316 +-
be/src/runtime/bufferpool/buffer_pool.cc | 691 ++-
be/src/runtime/bufferpool/buffer_pool.h | 724 ++-
be/src/runtime/bufferpool/buffer_pool_counters.h | 20 +-
be/src/runtime/bufferpool/buffer_pool_internal.h | 513 +-
be/src/runtime/bufferpool/free_list.h | 109 +-
be/src/runtime/bufferpool/reservation_tracker.cc | 610 ++-
be/src/runtime/bufferpool/reservation_tracker.h | 407 +-
.../bufferpool/reservation_tracker_counters.h | 14 +-
be/src/runtime/bufferpool/reservation_util.cc | 15 +-
be/src/runtime/bufferpool/reservation_util.h | 80 +-
be/src/runtime/bufferpool/suballocator.cc | 352 +-
be/src/runtime/bufferpool/suballocator.h | 266 +-
be/src/runtime/bufferpool/system_allocator.cc | 206 +-
be/src/runtime/bufferpool/system_allocator.h | 29 +-
be/src/runtime/cache/cache_utils.h | 2 +-
be/src/runtime/cache/result_cache.cpp | 17 +-
be/src/runtime/cache/result_cache.h | 2 +-
be/src/runtime/cache/result_node.cpp | 31 +-
be/src/runtime/cache/result_node.h | 2 +-
be/src/runtime/client_cache.cpp | 58 +-
be/src/runtime/client_cache.h | 95 +-
be/src/runtime/data_spliter.cpp | 86 +-
be/src/runtime/data_spliter.h | 24 +-
be/src/runtime/data_stream_mgr.cpp | 67 +-
be/src/runtime/data_stream_mgr.h | 36 +-
be/src/runtime/data_stream_recvr.cc | 112 +-
be/src/runtime/data_stream_recvr.h | 15 +-
be/src/runtime/data_stream_sender.cpp | 221 +-
be/src/runtime/data_stream_sender.h | 35 +-
be/src/runtime/datetime_value.cpp | 211 +-
be/src/runtime/datetime_value.h | 214 +-
be/src/runtime/decimal_value.cpp | 158 +-
be/src/runtime/decimal_value.h | 186 +-
be/src/runtime/decimalv2_value.cpp | 165 +-
be/src/runtime/decimalv2_value.h | 224 +-
be/src/runtime/descriptor_helper.h | 20 +-
be/src/runtime/descriptors.cpp | 205 +-
be/src/runtime/descriptors.h | 242 +-
be/src/runtime/disk_io_mgr.cc | 167 +-
be/src/runtime/disk_io_mgr.h | 83 +-
be/src/runtime/disk_io_mgr_internal.h | 33 +-
be/src/runtime/disk_io_mgr_reader_context.cc | 92 +-
be/src/runtime/disk_io_mgr_scan_range.cc | 36 +-
be/src/runtime/dpp_sink.cpp | 247 +-
be/src/runtime/dpp_sink.h | 36 +-
be/src/runtime/dpp_sink_internal.cpp | 91 +-
be/src/runtime/dpp_sink_internal.h | 140 +-
be/src/runtime/dpp_writer.cpp | 50 +-
be/src/runtime/dpp_writer.h | 6 +-
be/src/runtime/etl_job_mgr.cpp | 78 +-
be/src/runtime/etl_job_mgr.h | 20 +-
be/src/runtime/exec_env.cpp | 3 +-
be/src/runtime/exec_env.h | 1 -
be/src/runtime/exec_env_init.cpp | 108 +-
be/src/runtime/export_sink.cpp | 228 +-
be/src/runtime/export_sink.h | 8 +-
be/src/runtime/export_task_mgr.cpp | 55 +-
be/src/runtime/export_task_mgr.h | 7 +-
be/src/runtime/external_scan_context_mgr.cpp | 33 +-
be/src/runtime/external_scan_context_mgr.h | 12 +-
be/src/runtime/file_result_writer.cpp | 236 +-
be/src/runtime/file_result_writer.h | 17 +-
be/src/runtime/fragment_mgr.cpp | 263 +-
be/src/runtime/fragment_mgr.h | 22 +-
be/src/runtime/heartbeat_flags.h | 12 +-
be/src/runtime/initial_reservations.cc | 89 +-
be/src/runtime/initial_reservations.h | 66 +-
be/src/runtime/large_int_value.cpp | 8 +-
be/src/runtime/large_int_value.h | 17 +-
be/src/runtime/load_channel.cpp | 28 +-
be/src/runtime/load_channel.h | 11 +-
be/src/runtime/load_channel_mgr.cpp | 61 +-
be/src/runtime/load_channel_mgr.h | 11 +-
be/src/runtime/load_path_mgr.cpp | 59 +-
be/src/runtime/load_path_mgr.h | 16 +-
be/src/runtime/mem_pool.cpp | 37 +-
be/src/runtime/mem_pool.h | 33 +-
be/src/runtime/mem_tracker.cpp | 708 ++-
be/src/runtime/mem_tracker.h | 939 ++--
be/src/runtime/memory/chunk.h | 2 +-
be/src/runtime/memory/chunk_allocator.cpp | 17 +-
be/src/runtime/memory/chunk_allocator.h | 11 +-
be/src/runtime/memory/system_allocator.cpp | 13 +-
be/src/runtime/memory/system_allocator.h | 6 +-
be/src/runtime/memory_scratch_sink.cpp | 24 +-
be/src/runtime/memory_scratch_sink.h | 14 +-
be/src/runtime/message_body_sink.cpp | 16 +-
be/src/runtime/message_body_sink.h | 18 +-
be/src/runtime/multi_precision.h | 3 +-
be/src/runtime/mysql_result_writer.cpp | 42 +-
be/src/runtime/mysql_result_writer.h | 8 +-
be/src/runtime/mysql_table_sink.cpp | 8 +-
be/src/runtime/mysql_table_sink.h | 9 +-
be/src/runtime/mysql_table_writer.cpp | 40 +-
be/src/runtime/mysql_table_writer.h | 14 +-
be/src/runtime/plan_fragment_executor.cpp | 145 +-
be/src/runtime/plan_fragment_executor.h | 70 +-
be/src/runtime/primitive_type.cpp | 18 +-
be/src/runtime/primitive_type.h | 46 +-
be/src/runtime/qsorter.cpp | 35 +-
be/src/runtime/qsorter.h | 16 +-
be/src/runtime/query_statistics.cpp | 2 +-
be/src/runtime/query_statistics.h | 29 +-
be/src/runtime/raw_value.cpp | 116 +-
be/src/runtime/raw_value.h | 73 +-
be/src/runtime/raw_value_ir.cpp | 18 +-
be/src/runtime/record_batch_queue.cpp | 2 +-
be/src/runtime/record_batch_queue.h | 3 +-
be/src/runtime/result_buffer_mgr.cpp | 36 +-
be/src/runtime/result_buffer_mgr.h | 14 +-
be/src/runtime/result_queue_mgr.cpp | 11 +-
be/src/runtime/result_queue_mgr.h | 11 +-
be/src/runtime/result_sink.cpp | 59 +-
be/src/runtime/result_sink.h | 12 +-
be/src/runtime/result_writer.h | 9 +-
be/src/runtime/routine_load/data_consumer.cpp | 126 +-
be/src/runtime/routine_load/data_consumer.h | 83 +-
.../runtime/routine_load/data_consumer_group.cpp | 109 +-
be/src/runtime/routine_load/data_consumer_group.h | 30 +-
be/src/runtime/routine_load/data_consumer_pool.cpp | 69 +-
be/src/runtime/routine_load/data_consumer_pool.h | 13 +-
be/src/runtime/routine_load/kafka_consumer_pipe.h | 18 +-
.../routine_load/routine_load_task_executor.cpp | 206 +-
.../routine_load/routine_load_task_executor.h | 3 +-
be/src/runtime/row_batch.cpp | 176 +-
be/src/runtime/row_batch.h | 146 +-
be/src/runtime/runtime_state.cpp | 222 +-
be/src/runtime/runtime_state.h | 317 +-
be/src/runtime/small_file_mgr.cpp | 51 +-
be/src/runtime/small_file_mgr.h | 19 +-
be/src/runtime/snapshot_loader.cpp | 378 +-
be/src/runtime/snapshot_loader.h | 110 +-
be/src/runtime/sorted_run_merger.cc | 47 +-
be/src/runtime/sorted_run_merger.h | 4 +-
be/src/runtime/sorter.h | 19 +-
be/src/runtime/spill_sorter.cc | 301 +-
be/src/runtime/spill_sorter.h | 12 +-
be/src/runtime/stream_load/load_stream_mgr.cpp | 2 +-
be/src/runtime/stream_load/load_stream_mgr.h | 7 +-
be/src/runtime/stream_load/stream_load_context.cpp | 28 +-
be/src/runtime/stream_load/stream_load_context.h | 32 +-
.../runtime/stream_load/stream_load_executor.cpp | 20 +-
be/src/runtime/stream_load/stream_load_executor.h | 5 +-
be/src/runtime/stream_load/stream_load_pipe.h | 53 +-
be/src/runtime/string_value.cpp | 6 +-
be/src/runtime/string_value.h | 53 +-
be/src/runtime/string_value_ir.cpp | 5 +-
be/src/runtime/tablets_channel.cpp | 41 +-
be/src/runtime/tablets_channel.h | 21 +-
be/src/runtime/test_env.cc | 18 +-
be/src/runtime/test_env.h | 21 +-
be/src/runtime/thread_resource_mgr.cpp | 15 +-
be/src/runtime/thread_resource_mgr.h | 37 +-
be/src/runtime/tmp_file_mgr.cc | 50 +-
be/src/runtime/tmp_file_mgr.h | 33 +-
be/src/runtime/tuple.cpp | 48 +-
be/src/runtime/tuple.h | 32 +-
be/src/runtime/tuple_row.cpp | 3 +-
be/src/runtime/tuple_row.h | 22 +-
be/src/runtime/types.cpp | 21 +-
be/src/runtime/types.h | 33 +-
be/src/runtime/user_function_cache.cpp | 69 +-
be/src/runtime/user_function_cache.h | 23 +-
be/src/runtime/vectorized_row_batch.cpp | 22 +-
be/src/runtime/vectorized_row_batch.h | 71 +-
be/src/service/backend_options.cpp | 11 +-
be/src/service/backend_options.h | 5 +-
be/src/service/backend_service.cpp | 154 +-
be/src/service/backend_service.h | 43 +-
be/src/service/brpc.h | 12 +-
be/src/service/brpc_service.cpp | 14 +-
be/src/service/brpc_service.h | 3 +-
be/src/service/doris_main.cpp | 5 +-
be/src/service/http_service.cpp | 66 +-
be/src/service/http_service.h | 2 +-
be/src/service/internal_service.cpp | 200 +-
be/src/service/internal_service.h | 70 +-
be/src/testutil/desc_tbl_builder.cc | 14 +-
be/src/testutil/desc_tbl_builder.h | 22 +-
be/src/testutil/function_utils.cpp | 19 +-
be/src/testutil/function_utils.h | 10 +-
be/src/tools/meta_tool.cpp | 82 +-
be/src/udf/uda_test_harness.h | 158 +-
be/src/udf/udf.cpp | 134 +-
be/src/udf/udf.h | 129 +-
be/src/udf/udf_debug.h | 10 +-
be/src/udf/udf_internal.h | 86 +-
be/src/udf/udf_ir.cpp | 2 +-
be/src/util/aes_util.cpp | 71 +-
be/src/util/aes_util.h | 24 +-
be/src/util/aligned_new.h | 40 +-
be/src/util/alignment.h | 3 +-
be/src/util/arrow/row_batch.cpp | 176 +-
be/src/util/arrow/row_batch.h | 32 +-
be/src/util/arrow/row_block.cpp | 65 +-
be/src/util/arrow/row_block.h | 22 +-
be/src/util/arrow/utils.cpp | 2 +-
be/src/util/arrow/utils.h | 6 +-
be/src/util/barrier.h | 9 +-
be/src/util/bfd_parser.cpp | 53 +-
be/src/util/bfd_parser.h | 16 +-
be/src/util/bit_stream_utils.h | 190 +-
be/src/util/bit_stream_utils.inline.h | 287 +-
be/src/util/bit_util.h | 340 +-
be/src/util/bitmap.cpp | 48 +-
be/src/util/bitmap.h | 156 +-
be/src/util/bitmap_value.h | 318 +-
be/src/util/block_compression.cpp | 142 +-
be/src/util/block_compression.h | 8 +-
be/src/util/broker_load_error_hub.cpp | 22 +-
be/src/util/broker_load_error_hub.h | 15 +-
be/src/util/brpc_stub_cache.cpp | 2 +-
be/src/util/brpc_stub_cache.h | 8 +-
be/src/util/byte_buffer.h | 9 +-
be/src/util/cgroup_util.cpp | 55 +-
be/src/util/cgroup_util.h | 6 +-
be/src/util/cidr.cpp | 13 +-
be/src/util/coding.cpp | 4 +-
be/src/util/coding.h | 25 +-
be/src/util/condition_variable.cpp | 12 +-
be/src/util/condition_variable.h | 3 +-
be/src/util/core_local.cpp | 13 +-
be/src/util/core_local.h | 21 +-
be/src/util/countdown_latch.h | 27 +-
be/src/util/cpu_info.cpp | 475 +-
be/src/util/cpu_info.h | 372 +-
be/src/util/crc32c.cpp | 451 +-
be/src/util/crc32c.h | 1 +
be/src/util/date_func.cpp | 26 +-
be/src/util/date_func.h | 4 +-
be/src/util/debug/leak_annotations.h | 73 +-
be/src/util/debug/leakcheck_disabler.h | 12 +-
be/src/util/debug_counters.h | 13 +-
be/src/util/debug_util.cpp | 40 +-
be/src/util/debug_util.h | 8 +-
be/src/util/defer_op.h | 12 +-
be/src/util/disk_info.cpp | 6 +-
be/src/util/disk_info.h | 10 +-
be/src/util/doris_metrics.cpp | 106 +-
be/src/util/doris_metrics.h | 29 +-
be/src/util/dummy_runtime_profile.h | 6 +-
be/src/util/dynamic_util.cpp | 5 +-
be/src/util/dynamic_util.h | 3 +-
be/src/util/easy_json.cc | 202 +-
be/src/util/easy_json.h | 280 +-
be/src/util/errno.cpp | 6 +-
be/src/util/errno.h | 4 +-
be/src/util/error_util.cc | 6 +-
be/src/util/error_util.h | 4 +-
be/src/util/faststring.cc | 71 +-
be/src/util/faststring.h | 393 +-
be/src/util/file_cache.cpp | 26 +-
be/src/util/file_cache.h | 9 +-
be/src/util/file_utils.cpp | 39 +-
be/src/util/file_utils.h | 36 +-
be/src/util/filesystem_util.cc | 35 +-
be/src/util/filesystem_util.h | 5 +-
be/src/util/frame_of_reference_coding.cpp | 77 +-
be/src/util/frame_of_reference_coding.h | 46 +-
be/src/util/hash_util_ir.cpp | 8 +-
be/src/util/internal_queue.h | 392 +-
be/src/util/json_util.cpp | 2 +-
be/src/util/json_util.h | 14 +-
be/src/util/load_error_hub.cpp | 22 +-
be/src/util/load_error_hub.h | 18 +-
be/src/util/logging.h | 3 +-
be/src/util/md5.cpp | 6 +-
be/src/util/md5.h | 11 +-
be/src/util/mem_info.cpp | 20 +-
be/src/util/mem_info.h | 4 +-
be/src/util/mem_range.h | 24 +-
be/src/util/metrics.cpp | 51 +-
be/src/util/metrics.h | 164 +-
be/src/util/minizip/ioapi.h | 181 +-
be/src/util/minizip/unzip.h | 220 +-
be/src/util/monotime.cpp | 56 +-
be/src/util/monotime.h | 36 +-
be/src/util/murmur_hash3.cpp | 665 ++-
be/src/util/murmur_hash3.h | 2 +-
be/src/util/mustache/mustache.cc | 640 +--
be/src/util/mustache/mustache.h | 7 +-
be/src/util/mutex.cpp | 103 +-
be/src/util/mutex.h | 25 +-
be/src/util/mysql_global.h | 47 +-
be/src/util/mysql_load_error_hub.cpp | 30 +-
be/src/util/mysql_load_error_hub.h | 31 +-
be/src/util/mysql_row_buffer.cpp | 13 +-
be/src/util/mysql_row_buffer.h | 25 +-
be/src/util/network_util.cpp | 25 +-
be/src/util/network_util.h | 6 +-
be/src/util/null_load_error_hub.cpp | 7 +-
be/src/util/null_load_error_hub.h | 6 +-
be/src/util/once.h | 11 +-
be/src/util/os_info.cpp | 67 +-
be/src/util/os_info.h | 48 +-
be/src/util/os_util.cpp | 3 +-
be/src/util/parse_util.cpp | 13 +-
be/src/util/parse_util.h | 4 +-
be/src/util/path_builder.cpp | 5 +-
be/src/util/path_builder.h | 3 +-
be/src/util/perf_counters.cpp | 21 +-
be/src/util/perf_counters.h | 6 +-
be/src/util/pprof_utils.cpp | 42 +-
be/src/util/pprof_utils.h | 15 +-
be/src/util/pretty_printer.h | 35 +-
be/src/util/progress_updater.cpp | 34 +-
be/src/util/progress_updater.h | 20 +-
be/src/util/radix_sort.h | 125 +-
be/src/util/random.h | 13 +-
be/src/util/ref_count_closure.h | 19 +-
be/src/util/rle_encoding.h | 736 ++-
be/src/util/runtime_profile.cpp | 3 +-
be/src/util/runtime_profile.h | 173 +-
be/src/util/scoped_cleanup.h | 11 +-
be/src/util/simdutf8check.h | 612 +--
be/src/util/slice.cpp | 8 +-
be/src/util/slice.h | 78 +-
be/src/util/spinlock.cc | 1 -
be/src/util/spinlock.h | 9 +-
be/src/util/stack_util.cpp | 4 +-
be/src/util/stack_util.h | 2 +-
be/src/util/static_asserts.cpp | 4 +-
be/src/util/streaming_sampler.h | 40 +-
be/src/util/string_parser.cpp | 4 +-
be/src/util/string_util.cpp | 2 +-
be/src/util/string_util.h | 18 +-
be/src/util/symbols_util.cpp | 24 +-
be/src/util/symbols_util.h | 9 +-
be/src/util/system_metrics.cpp | 142 +-
be/src/util/system_metrics.h | 27 +-
be/src/util/tdigest.h | 120 +-
be/src/util/template_util.h | 2 +-
be/src/util/thread.cpp | 9 +-
be/src/util/threadpool.cpp | 167 +-
be/src/util/threadpool.h | 22 +-
be/src/util/thrift_client.cpp | 9 +-
be/src/util/thrift_client.h | 85 +-
be/src/util/thrift_rpc_helper.cpp | 75 +-
be/src/util/thrift_rpc_helper.h | 23 +-
be/src/util/thrift_server.cpp | 129 +-
be/src/util/thrift_server.h | 21 +-
be/src/util/thrift_util.h | 35 +-
be/src/util/time.cpp | 115 +-
be/src/util/time.h | 62 +-
be/src/util/timezone_utils.cpp | 7 +-
be/src/util/timezone_utils.h | 5 +-
be/src/util/trace.cpp | 311 +-
be/src/util/trace.h | 332 +-
be/src/util/trace_metrics.cpp | 39 +-
be/src/util/trace_metrics.h | 66 +-
be/src/util/tuple_row_compare.cpp | 3 +-
be/src/util/tuple_row_compare.h | 67 +-
be/src/util/types.h | 9 +-
be/src/util/uid_util.cpp | 2 +-
be/src/util/uid_util.h | 69 +-
be/src/util/url_coding.cpp | 3 +-
be/src/util/url_coding.h | 10 +-
be/src/util/url_parser.cpp | 12 +-
be/src/util/url_parser.h | 24 +-
be/src/util/utf8_check.cpp | 118 +-
be/src/util/utf8_check.h | 4 +-
be/src/util/uuid_generator.h | 10 +-
be/src/util/zip_util.cpp | 30 +-
be/src/util/zip_util.h | 18 +-
be/src/util/zlib.cpp | 129 +-
be/src/util/zlib.h | 2 +-
be/test/agent/agent_server_test.cpp | 29 +-
be/test/agent/cgroups_mgr_test.cpp | 45 +-
be/test/agent/heartbeat_server_test.cpp | 22 +-
be/test/agent/mock_pusher.h | 6 +-
be/test/agent/mock_task_worker_pool.h | 6 +-
be/test/agent/mock_utils.h | 58 +-
be/test/agent/utils_test.cpp | 10 +-
be/test/common/config_test.cpp | 10 +-
be/test/common/resource_tls_test.cpp | 11 +-
be/test/common/status_test.cpp | 5 +-
be/test/env/env_posix_test.cpp | 17 +-
be/test/exec/broker_reader_test.cpp | 12 +-
be/test/exec/broker_scan_node_test.cpp | 27 +-
be/test/exec/broker_scanner_test.cpp | 24 +-
be/test/exec/buffered_reader_test.cpp | 11 +-
be/test/exec/csv_scan_bench_test.cpp | 19 +-
be/test/exec/csv_scan_node_test.cpp | 34 +-
be/test/exec/csv_scanner_test.cpp | 15 +-
be/test/exec/es_http_scan_node_test.cpp | 27 +-
be/test/exec/es_predicate_test.cpp | 13 +-
be/test/exec/es_query_builder_test.cpp | 169 +-
be/test/exec/es_scan_node_test.cpp | 30 +-
be/test/exec/es_scan_reader_test.cpp | 55 +-
be/test/exec/hash_table_test.cpp | 30 +-
be/test/exec/json_scanner_test.cpp | 64 +-
be/test/exec/json_scanner_test_with_jsonpath.cpp | 70 +-
be/test/exec/mysql_scan_node_test.cpp | 22 +-
be/test/exec/mysql_scanner_test.cpp | 14 +-
be/test/exec/new_olap_scan_node_test.cpp | 29 +-
be/test/exec/olap_common_test.cpp | 144 +-
be/test/exec/olap_scan_node_test.cpp | 34 +-
be/test/exec/olap_scanner_test.cpp | 13 +-
be/test/exec/orc_scanner_test.cpp | 269 +-
be/test/exec/parquet_scanner_test.cpp | 80 +-
be/test/exec/plain_text_line_reader_bzip_test.cpp | 19 +-
be/test/exec/plain_text_line_reader_gzip_test.cpp | 36 +-
.../exec/plain_text_line_reader_lz4frame_test.cpp | 31 +-
be/test/exec/plain_text_line_reader_lzop_test.cpp | 22 +-
.../plain_text_line_reader_uncompressed_test.cpp | 18 +-
be/test/exec/schema_scan_node_test.cpp | 28 +-
.../schema_scanner/schema_authors_scanner_test.cpp | 37 +-
.../schema_charsets_scanner_test.cpp | 35 +-
.../schema_collations_scanner_test.cpp | 35 +-
.../schema_scanner/schema_columns_scanner_test.cpp | 62 +-
.../schema_create_table_scanner_test.cpp | 60 +-
.../schema_scanner/schema_engines_scanner_test.cpp | 33 +-
.../schema_open_tables_scanner_test.cpp | 58 +-
.../schema_schemata_scanner_test.cpp | 46 +-
.../schema_table_names_scanner_test.cpp | 52 +-
.../schema_scanner/schema_tables_scanner_test.cpp | 58 +-
.../schema_variables_scanner_test.cpp | 33 +-
be/test/exec/schema_scanner_test.cpp | 31 +-
be/test/exec/set_executor_test.cpp | 14 +-
be/test/exec/tablet_info_test.cpp | 25 +-
be/test/exprs/binary_predicate_test.cpp | 52 +-
be/test/exprs/bitmap_function_test.cpp | 112 +-
be/test/exprs/encryption_functions_test.cpp | 19 +-
be/test/exprs/hll_function_test.cpp | 15 +-
be/test/exprs/hybrid_set_test.cpp | 16 +-
be/test/exprs/in_op_test.cpp | 34 +-
be/test/exprs/in_predicate_test.cpp | 14 +-
be/test/exprs/json_function_test.cpp | 137 +-
be/test/exprs/math_functions_test.cpp | 8 +-
be/test/exprs/percentile_approx_test.cpp | 23 +-
be/test/exprs/string_functions_test.cpp | 103 +-
be/test/exprs/timestamp_functions_test.cpp | 74 +-
be/test/geo/geo_functions_test.cpp | 33 +-
be/test/geo/geo_types_test.cpp | 11 +-
be/test/geo/wkt_parse_test.cpp | 8 +-
be/test/http/http_client_test.cpp | 14 +-
be/test/http/http_utils_test.cpp | 15 +-
be/test/http/message_body_sink_test.cpp | 15 +-
be/test/http/metrics_action_test.cpp | 38 +-
be/test/http/stream_load_test.cpp | 27 +-
be/test/olap/aggregate_func_test.cpp | 29 +-
be/test/olap/bit_field_test.cpp | 60 +-
be/test/olap/bloom_filter_index_test.cpp | 12 +-
be/test/olap/bloom_filter_test.cpp | 10 +-
be/test/olap/byte_buffer_test.cpp | 27 +-
be/test/olap/column_reader_test.cpp | 1069 ++--
be/test/olap/column_vector_test.cpp | 44 +-
be/test/olap/comparison_predicate_test.cpp | 376 +-
be/test/olap/cumulative_compaction_policy_test.cpp | 190 +-
be/test/olap/decimal12_test.cpp | 10 +-
be/test/olap/delete_handler_test.cpp | 190 +-
be/test/olap/delta_writer_test.cpp | 140 +-
be/test/olap/file_helper_test.cpp | 32 +-
be/test/olap/file_utils_test.cpp | 81 +-
be/test/olap/fs/file_block_manager_test.cpp | 9 +-
be/test/olap/generic_iterators_test.cpp | 17 +-
be/test/olap/hll_test.cpp | 4 +-
be/test/olap/in_list_predicate_test.cpp | 334 +-
be/test/olap/key_coder_test.cpp | 22 +-
be/test/olap/lru_cache_test.cpp | 61 +-
be/test/olap/memtable_flush_executor_test.cpp | 23 +-
be/test/olap/null_predicate_test.cpp | 172 +-
be/test/olap/olap_meta_test.cpp | 25 +-
be/test/olap/options_test.cpp | 12 +-
be/test/olap/page_cache_test.cpp | 8 +-
be/test/olap/push_handler_test.cpp | 36 +-
be/test/olap/row_block_test.cpp | 23 +-
be/test/olap/row_block_v2_test.cpp | 13 +-
be/test/olap/row_cursor_test.cpp | 9 +-
be/test/olap/rowset/alpha_rowset_test.cpp | 62 +-
be/test/olap/rowset/beta_rowset_test.cpp | 23 +-
be/test/olap/rowset/rowset_converter_test.cpp | 71 +-
be/test/olap/rowset/rowset_meta_manager_test.cpp | 27 +-
be/test/olap/rowset/rowset_meta_test.cpp | 19 +-
.../rowset/segment_v2/binary_dict_page_test.cpp | 43 +-
.../rowset/segment_v2/binary_plain_page_test.cpp | 39 +-
.../rowset/segment_v2/binary_prefix_page_test.cpp | 26 +-
.../olap/rowset/segment_v2/bitmap_index_test.cpp | 34 +-
.../rowset/segment_v2/bitshuffle_page_test.cpp | 114 +-
.../rowset/segment_v2/block_bloom_filter_test.cpp | 8 +-
.../bloom_filter_index_reader_writer_test.cpp | 67 +-
.../rowset/segment_v2/bloom_filter_page_test.cpp | 49 +-
.../segment_v2/column_reader_writer_test.cpp | 132 +-
.../olap/rowset/segment_v2/encoding_info_test.cpp | 11 +-
.../segment_v2/frame_of_reference_page_test.cpp | 75 +-
.../rowset/segment_v2/ordinal_page_index_test.cpp | 11 +-
be/test/olap/rowset/segment_v2/plain_page_test.cpp | 157 +-
be/test/olap/rowset/segment_v2/rle_page_test.cpp | 52 +-
be/test/olap/rowset/segment_v2/row_ranges_test.cpp | 11 +-
be/test/olap/rowset/segment_v2/segment_test.cpp | 164 +-
.../olap/rowset/segment_v2/zone_map_index_test.cpp | 13 +-
.../rowset/unique_rowset_id_generator_test.cpp | 43 +-
be/test/olap/run_length_byte_test.cpp | 277 +-
be/test/olap/run_length_integer_test.cpp | 1106 +++-
be/test/olap/schema_change_test.cpp | 219 +-
be/test/olap/selection_vector_test.cpp | 11 +-
be/test/olap/serialize_test.cpp | 11 +-
be/test/olap/short_key_index_test.cpp | 15 +-
be/test/olap/skiplist_test.cpp | 86 +-
be/test/olap/storage_types_test.cpp | 32 +-
be/test/olap/stream_index_test.cpp | 104 +-
be/test/olap/tablet_meta_manager_test.cpp | 17 +-
be/test/olap/tablet_meta_test.cpp | 11 +-
be/test/olap/tablet_mgr_test.cpp | 37 +-
be/test/olap/tablet_schema_helper.h | 24 +-
be/test/olap/tablet_test.cpp | 24 +-
be/test/olap/timestamped_version_tracker_test.cpp | 89 +-
be/test/olap/txn_manager_test.cpp | 150 +-
be/test/plugin/example/plugin_example.cpp | 17 +-
be/test/plugin/plugin_loader_test.cpp | 41 +-
be/test/plugin/plugin_mgr_test.cpp | 33 +-
be/test/plugin/plugin_zip_test.cpp | 43 +-
be/test/runtime/buffer_control_block_test.cpp | 17 +-
be/test/runtime/buffered_block_mgr2_test.cpp | 49 +-
be/test/runtime/buffered_tuple_stream2_test.cpp | 108 +-
be/test/runtime/buffered_tuple_stream_test.cpp | 82 +-
be/test/runtime/cache/partition_cache_test.cpp | 54 +-
be/test/runtime/data_spliter_test.cpp | 25 +-
be/test/runtime/data_stream_test.cpp | 4 +-
be/test/runtime/datetime_value_test.cpp | 122 +-
be/test/runtime/decimal_value_test.cpp | 45 +-
be/test/runtime/decimalv2_value_test.cpp | 38 +-
be/test/runtime/disk_io_mgr_test.cpp | 188 +-
be/test/runtime/dpp_sink_internal_test.cpp | 19 +-
be/test/runtime/dpp_sink_test.cpp | 29 +-
be/test/runtime/etl_job_mgr_test.cpp | 24 +-
be/test/runtime/export_task_mgr_test.cpp | 24 +-
be/test/runtime/external_scan_context_mgr_test.cpp | 14 +-
be/test/runtime/fragment_mgr_test.cpp | 5 +-
be/test/runtime/free_list_test.cpp | 9 +-
be/test/runtime/heartbeat_flags_test.cpp | 8 +-
be/test/runtime/kafka_consumer_pipe_test.cpp | 27 +-
be/test/runtime/large_int_value_test.cpp | 14 +-
be/test/runtime/load_channel_mgr_test.cpp | 50 +-
be/test/runtime/mem_limit_test.cpp | 13 +-
be/test/runtime/mem_pool_test.cpp | 16 +-
be/test/runtime/memory/chunk_allocator_test.cpp | 2 +-
be/test/runtime/memory/system_allocator_test.cpp | 4 +-
be/test/runtime/memory_scratch_sink_test.cpp | 20 +-
be/test/runtime/qsorter_test.cpp | 37 +-
be/test/runtime/raw_value_test.cpp | 8 +-
be/test/runtime/result_buffer_mgr_test.cpp | 20 +-
be/test/runtime/result_queue_mgr_test.cpp | 18 +-
be/test/runtime/result_sink_test.cpp | 38 +-
.../runtime/routine_load_task_executor_test.cpp | 22 +-
be/test/runtime/small_file_mgr_test.cpp | 17 +-
be/test/runtime/snapshot_loader_test.cpp | 28 +-
be/test/runtime/sorter_test.cpp | 112 +-
be/test/runtime/string_buffer_test.cpp | 6 +-
be/test/runtime/string_value_test.cpp | 6 +-
.../test_data/user_function_cache/lib/my_add.cc | 6 +-
be/test/runtime/thread_resource_mgr_test.cpp | 26 +-
be/test/runtime/tmp_file_mgr_test.cpp | 26 +-
be/test/runtime/user_function_cache_test.cpp | 58 +-
be/test/udf/uda_test.cpp | 35 +-
be/test/udf/udf_test.cpp | 52 +-
be/test/util/aes_util_test.cpp | 31 +-
be/test/util/arrow/arrow_row_batch_test.cpp | 3 +-
be/test/util/arrow/arrow_row_block_test.cpp | 4 +-
be/test/util/arrow/arrow_work_flow_test.cpp | 7 +-
be/test/util/bit_stream_utils_test.cpp | 53 +-
be/test/util/bit_util_test.cpp | 13 +-
be/test/util/bitmap_test.cpp | 14 +-
be/test/util/bitmap_value_test.cpp | 29 +-
be/test/util/block_compression_test.cpp | 22 +-
be/test/util/blocking_queue_test.cpp | 18 +-
be/test/util/brpc_stub_cache_test.cpp | 7 +-
be/test/util/byte_buffer_test2.cpp | 9 +-
be/test/util/cgroup_util_test.cpp | 13 +-
be/test/util/cidr_test.cpp | 7 +-
be/test/util/coding_test.cpp | 10 +-
be/test/util/core_local_test.cpp | 10 +-
be/test/util/countdown_latch_test.cpp | 7 +-
be/test/util/counter_cond_variable_test.cpp | 9 +-
be/test/util/crc32c_test.cpp | 35 +-
be/test/util/decompress_test.cpp | 33 +-
be/test/util/doris_metrics_test.cpp | 46 +-
be/test/util/easy_json-test.cpp | 106 +-
be/test/util/faststring_test.cpp | 163 +-
be/test/util/file_cache_test.cpp | 4 +-
be/test/util/filesystem_util_test.cpp | 11 +-
be/test/util/frame_of_reference_coding_test.cpp | 6 +-
be/test/util/http_channel_test.cpp | 3 +-
be/test/util/internal_queue_test.cpp | 35 +-
be/test/util/json_util_test.cpp | 27 +-
be/test/util/lru_cache_util_test.cpp | 10 +-
be/test/util/md5_test.cpp | 6 +-
be/test/util/monotime_test.cpp | 11 +-
be/test/util/new_metrics_test.cpp | 70 +-
be/test/util/parse_util_test.cpp | 7 +-
be/test/util/path_trie_test.cpp | 8 +-
be/test/util/path_util_test.cpp | 4 +-
be/test/util/perf_counters_test.cpp | 20 +-
be/test/util/radix_sort_test.cpp | 15 +-
be/test/util/rle_encoding_test.cpp | 611 ++-
be/test/util/runtime_profile_test.cpp | 61 +-
be/test/util/scoped_cleanup_test.cpp | 5 +-
be/test/util/string_parser_test.cpp | 100 +-
be/test/util/string_util_test.cpp | 7 +-
be/test/util/tdigest_test.cpp | 16 +-
be/test/util/thread_pool_test.cpp | 10 +-
be/test/util/threadpool_test.cpp | 227 +-
be/test/util/trace_test.cpp | 129 +-
be/test/util/types_test.cpp | 9 +-
be/test/util/uid_util_test.cpp | 9 +-
be/test/util/url_coding_test.cpp | 11 +-
be/test/util/utf8_check_test.cpp | 124 +-
be/test/util/zip_util_test.cpp | 25 +-
1331 files changed, 63849 insertions(+), 69815 deletions(-)
diff --git a/be/src/agent/agent_server.cpp b/be/src/agent/agent_server.cpp
index 57ed17a..fe0ce00 100644
--- a/be/src/agent/agent_server.cpp
+++ b/be/src/agent/agent_server.cpp
@@ -17,15 +17,14 @@
#include "agent/agent_server.h"
-#include <string>
-
#include <boost/filesystem.hpp>
+#include <string>
#include "agent/task_worker_pool.h"
#include "agent/topic_subscriber.h"
#include "agent/user_resource_listener.h"
-#include "common/status.h"
#include "common/logging.h"
+#include "common/status.h"
#include "gutil/strings/substitute.h"
#include "olap/snapshot_manager.h"
#include "runtime/etl_job_mgr.h"
@@ -35,10 +34,8 @@ using std::vector;
namespace doris {
-AgentServer::AgentServer(ExecEnv* exec_env, const TMasterInfo& master_info) :
- _exec_env(exec_env),
- _master_info(master_info),
- _topic_subscriber(new TopicSubscriber()) {
+AgentServer::AgentServer(ExecEnv* exec_env, const TMasterInfo& master_info)
+ : _exec_env(exec_env), _master_info(master_info), _topic_subscriber(new TopicSubscriber()) {
for (auto& path : exec_env->store_paths()) {
try {
string dpp_download_path_str = path.path + DPP_PREFIX;
@@ -55,11 +52,9 @@ AgentServer::AgentServer(ExecEnv* exec_env, const TMasterInfo& master_info) :
// to make code to be more readable.
#ifndef BE_TEST
-#define CREATE_AND_START_POOL(type, pool_name) \
- pool_name.reset(new TaskWorkerPool( \
- TaskWorkerPool::TaskWorkerType::type, \
- _exec_env, \
- master_info)); \
+#define CREATE_AND_START_POOL(type, pool_name) \
+ pool_name.reset( \
+ new TaskWorkerPool(TaskWorkerPool::TaskWorkerType::type, _exec_env, master_info)); \
pool_name->start();
#else
#define CREATE_AND_START_POOL(type, pool_name)
@@ -95,11 +90,12 @@ AgentServer::AgentServer(ExecEnv* exec_env, const TMasterInfo& master_info) :
#endif
}
-AgentServer::~AgentServer() { }
+AgentServer::~AgentServer() {}
// TODO(lingbin): each task in the batch may have it own status or FE must check and
// resend request when something is wrong(BE may need some logic to guarantee idempotence.
-void AgentServer::submit_tasks(TAgentResult& agent_result, const std::vector<TAgentTaskRequest>& tasks) {
+void AgentServer::submit_tasks(TAgentResult& agent_result,
+ const std::vector<TAgentTaskRequest>& tasks) {
Status ret_st;
// TODO check master_info here if it is the same with that of heartbeat rpc
@@ -126,27 +122,24 @@ void AgentServer::submit_tasks(TAgentResult& agent_result, const std::vector<TAg
// TODO(lingbin): It still too long, divided these task types into several categories
switch (task_type) {
- HANDLE_TYPE(TTaskType::CREATE, _create_tablet_workers, create_tablet_req);
- HANDLE_TYPE(TTaskType::DROP, _drop_tablet_workers, drop_tablet_req);
- HANDLE_TYPE(TTaskType::PUBLISH_VERSION, _publish_version_workers, publish_version_req);
- HANDLE_TYPE(TTaskType::CLEAR_TRANSACTION_TASK,
- _clear_transaction_task_workers,
- clear_transaction_task_req);
- HANDLE_TYPE(TTaskType::CLONE, _clone_workers, clone_req);
- HANDLE_TYPE(TTaskType::STORAGE_MEDIUM_MIGRATE,
- _storage_medium_migrate_workers,
- storage_medium_migrate_req);
- HANDLE_TYPE(TTaskType::CHECK_CONSISTENCY,
- _check_consistency_workers,
- check_consistency_req);
- HANDLE_TYPE(TTaskType::UPLOAD, _upload_workers, upload_req);
- HANDLE_TYPE(TTaskType::DOWNLOAD, _download_workers, download_req);
- HANDLE_TYPE(TTaskType::MAKE_SNAPSHOT, _make_snapshot_workers, snapshot_req);
- HANDLE_TYPE(TTaskType::RELEASE_SNAPSHOT, _release_snapshot_workers, release_snapshot_req);
- HANDLE_TYPE(TTaskType::MOVE, _move_dir_workers, move_dir_req);
- HANDLE_TYPE(TTaskType::UPDATE_TABLET_META_INFO,
- _update_tablet_meta_info_workers,
- update_tablet_meta_info_req);
+ HANDLE_TYPE(TTaskType::CREATE, _create_tablet_workers, create_tablet_req);
+ HANDLE_TYPE(TTaskType::DROP, _drop_tablet_workers, drop_tablet_req);
+ HANDLE_TYPE(TTaskType::PUBLISH_VERSION, _publish_version_workers, publish_version_req);
+ HANDLE_TYPE(TTaskType::CLEAR_TRANSACTION_TASK, _clear_transaction_task_workers,
+ clear_transaction_task_req);
+ HANDLE_TYPE(TTaskType::CLONE, _clone_workers, clone_req);
+ HANDLE_TYPE(TTaskType::STORAGE_MEDIUM_MIGRATE, _storage_medium_migrate_workers,
+ storage_medium_migrate_req);
+ HANDLE_TYPE(TTaskType::CHECK_CONSISTENCY, _check_consistency_workers,
+ check_consistency_req);
+ HANDLE_TYPE(TTaskType::UPLOAD, _upload_workers, upload_req);
+ HANDLE_TYPE(TTaskType::DOWNLOAD, _download_workers, download_req);
+ HANDLE_TYPE(TTaskType::MAKE_SNAPSHOT, _make_snapshot_workers, snapshot_req);
+ HANDLE_TYPE(TTaskType::RELEASE_SNAPSHOT, _release_snapshot_workers,
+ release_snapshot_req);
+ HANDLE_TYPE(TTaskType::MOVE, _move_dir_workers, move_dir_req);
+ HANDLE_TYPE(TTaskType::UPDATE_TABLET_META_INFO, _update_tablet_meta_info_workers,
+ update_tablet_meta_info_req);
case TTaskType::REALTIME_PUSH:
case TTaskType::PUSH:
@@ -155,16 +148,16 @@ void AgentServer::submit_tasks(TAgentResult& agent_result, const std::vector<TAg
"task(signature=$0) has wrong request member", signature));
break;
}
- if (task.push_req.push_type == TPushType::LOAD
- || task.push_req.push_type == TPushType::LOAD_DELETE
- || task.push_req.push_type == TPushType::LOAD_V2) {
+ if (task.push_req.push_type == TPushType::LOAD ||
+ task.push_req.push_type == TPushType::LOAD_DELETE ||
+ task.push_req.push_type == TPushType::LOAD_V2) {
_push_workers->submit_task(task);
} else if (task.push_req.push_type == TPushType::DELETE) {
_delete_workers->submit_task(task);
} else {
ret_st = Status::InvalidArgument(strings::Substitute(
- "task(signature=$0, type=$1, push_type=$2) has wrong push_type",
- signature, task_type, task.push_req.push_type));
+ "task(signature=$0, type=$1, push_type=$2) has wrong push_type", signature,
+ task_type, task.push_req.push_type));
}
break;
case TTaskType::ALTER:
@@ -184,7 +177,7 @@ void AgentServer::submit_tasks(TAgentResult& agent_result, const std::vector<TAg
if (!ret_st.ok()) {
LOG(WARNING) << "fail to submit task. reason: " << ret_st.get_error_msg()
- << ", task: " << task;
+ << ", task: " << task;
// For now, all tasks in the batch share one status, so if any task
// was failed to submit, we can only return error to FE(even when some
// tasks have already been successfully submitted).
@@ -211,8 +204,8 @@ void AgentServer::make_snapshot(TAgentResult& t_agent_result,
LOG(WARNING) << "fail to make_snapshot. tablet_id=" << snapshot_request.tablet_id
<< ", schema_hash=" << snapshot_request.schema_hash
<< ", error_code=" << err_code;
- ret_st = Status::RuntimeError(strings::Substitute(
- "fail to make_snapshot. err_code=$0", err_code));
+ ret_st = Status::RuntimeError(
+ strings::Substitute("fail to make_snapshot. err_code=$0", err_code));
} else {
LOG(INFO) << "success to make_snapshot. tablet_id=" << snapshot_request.tablet_id
<< ", schema_hash=" << snapshot_request.schema_hash
@@ -233,8 +226,8 @@ void AgentServer::release_snapshot(TAgentResult& t_agent_result, const std::stri
if (err_code != OLAP_SUCCESS) {
LOG(WARNING) << "failed to release_snapshot. snapshot_path: " << snapshot_path
<< ", err_code: " << err_code;
- ret_st = Status::RuntimeError(strings::Substitute(
- "fail to release_snapshot. err_code=$0", err_code));
+ ret_st = Status::RuntimeError(
+ strings::Substitute("fail to release_snapshot. err_code=$0", err_code));
} else {
LOG(INFO) << "success to release_snapshot. snapshot_path=" << snapshot_path
<< ", err_code=" << err_code;
@@ -268,9 +261,9 @@ void AgentServer::get_etl_status(TMiniLoadEtlStatusResult& t_agent_result,
LOG(WARNING) << "fail to get job state. [id=" << request.mini_load_id << "]";
}
- VLOG_RPC << "success to get job state. [id=" << request.mini_load_id << ", status="
- << t_agent_result.status.status_code << ", etl_state=" << t_agent_result.etl_state
- << ", files=";
+ VLOG_RPC << "success to get job state. [id=" << request.mini_load_id
+ << ", status=" << t_agent_result.status.status_code
+ << ", etl_state=" << t_agent_result.etl_state << ", files=";
for (auto& item : t_agent_result.file_map) {
VLOG_RPC << item.first << ":" << item.second << ";";
}
@@ -282,11 +275,11 @@ void AgentServer::delete_etl_files(TAgentResult& t_agent_result,
Status status = _exec_env->etl_job_mgr()->erase_job(request);
if (!status.ok()) {
LOG(WARNING) << "fail to delete etl files. because " << status.get_error_msg()
- << " with request " << request;
+ << " with request " << request;
}
VLOG_RPC << "success to delete etl files. request=" << request;
status.to_thrift(&t_agent_result.status);
}
-} // namespace doris
+} // namespace doris
diff --git a/be/src/agent/agent_server.h b/be/src/agent/agent_server.h
index 3ed126d..b4c0f15 100644
--- a/be/src/agent/agent_server.h
+++ b/be/src/agent/agent_server.h
@@ -89,7 +89,6 @@ private:
std::unique_ptr<TopicSubscriber> _topic_subscriber;
};
-} // end namespace doris
-
-#endif // DORIS_BE_SRC_AGENT_AGENT_SERVER_H
+} // end namespace doris
+#endif // DORIS_BE_SRC_AGENT_AGENT_SERVER_H
diff --git a/be/src/agent/cgroups_mgr.cpp b/be/src/agent/cgroups_mgr.cpp
index 128be59..53308fb 100644
--- a/be/src/agent/cgroups_mgr.cpp
+++ b/be/src/agent/cgroups_mgr.cpp
@@ -16,16 +16,19 @@
// under the License.
#include "agent/cgroups_mgr.h"
-#include <fstream>
-#include <future>
-#include <linux/magic.h>
-#include <map>
-#include <unistd.h>
+
#include <asm/unistd.h>
-#include <sstream>
+#include <linux/magic.h>
#include <sys/stat.h>
#include <sys/sysmacros.h>
#include <sys/vfs.h>
+#include <unistd.h>
+
+#include <fstream>
+#include <future>
+#include <map>
+#include <sstream>
+
#include "boost/filesystem.hpp"
#include "common/logging.h"
#include "olap/data_dir.h"
@@ -42,42 +45,39 @@ using apache::thrift::transport::TTransportException;
namespace doris {
-static CgroupsMgr *s_global_cg_mgr;
+static CgroupsMgr* s_global_cg_mgr;
const std::string CgroupsMgr::_s_system_user = "system";
const std::string CgroupsMgr::_s_system_group = "normal";
-std::map<TResourceType::type, std::string> CgroupsMgr::_s_resource_cgroups =
- {{TResourceType::type::TRESOURCE_CPU_SHARE, "cpu.shares"},
- {TResourceType::type::TRESOURCE_IO_SHARE, "blkio.weight"}};
+std::map<TResourceType::type, std::string> CgroupsMgr::_s_resource_cgroups = {
+ {TResourceType::type::TRESOURCE_CPU_SHARE, "cpu.shares"},
+ {TResourceType::type::TRESOURCE_IO_SHARE, "blkio.weight"}};
CgroupsMgr::CgroupsMgr(ExecEnv* exec_env, const string& root_cgroups_path)
- : _exec_env(exec_env),
- _root_cgroups_path(root_cgroups_path),
- _is_cgroups_init_success(false),
- _cur_version(-1) {
+ : _exec_env(exec_env),
+ _root_cgroups_path(root_cgroups_path),
+ _is_cgroups_init_success(false),
+ _cur_version(-1) {
if (s_global_cg_mgr == nullptr) {
s_global_cg_mgr = this;
}
}
-CgroupsMgr::~CgroupsMgr() {
-}
-
-AgentStatus CgroupsMgr::update_local_cgroups(const TFetchResourceResult& new_fetched_resource) {
+CgroupsMgr::~CgroupsMgr() {}
+AgentStatus CgroupsMgr::update_local_cgroups(const TFetchResourceResult& new_fetched_resource) {
std::lock_guard<std::mutex> lck(_update_cgroups_mtx);
if (!_is_cgroups_init_success) {
return AgentStatus::DORIS_ERROR;
}
-
if (_cur_version >= new_fetched_resource.resourceVersion) {
return AgentStatus::DORIS_SUCCESS;
}
- const std::map<std::string, TUserResource>& new_user_resource
- = new_fetched_resource.resourceByUser;
+ const std::map<std::string, TUserResource>& new_user_resource =
+ new_fetched_resource.resourceByUser;
if (!_local_users.empty()) {
std::set<std::string>::const_iterator old_it = _local_users.begin();
@@ -97,12 +97,11 @@ AgentStatus CgroupsMgr::update_local_cgroups(const TFetchResourceResult& new_fe
const std::map<std::string, int32_t>& level_share = new_it->second.shareByGroup;
std::map<std::string, int32_t> user_share;
const std::map<TResourceType::type, int32_t>& resource_share =
- new_it->second.resource.resourceByType;
+ new_it->second.resource.resourceByType;
std::map<TResourceType::type, int32_t>::const_iterator resource_it = resource_share.begin();
for (; resource_it != resource_share.end(); ++resource_it) {
if (_s_resource_cgroups.count(resource_it->first) > 0) {
- user_share[_s_resource_cgroups[resource_it->first]] =
- resource_it->second;
+ user_share[_s_resource_cgroups[resource_it->first]] = resource_it->second;
}
}
@@ -118,45 +117,42 @@ AgentStatus CgroupsMgr::update_local_cgroups(const TFetchResourceResult& new_fe
return AgentStatus::DORIS_SUCCESS;
}
-void CgroupsMgr::_config_user_disk_throttle(std::string user_name,
- const std::map<TResourceType::type, int32_t>& resource_share) {
- int64_t hdd_read_iops = _get_resource_value(TResourceType::type::TRESOURCE_HDD_READ_IOPS,
- resource_share);
- int64_t hdd_write_iops = _get_resource_value(TResourceType::type::TRESOURCE_HDD_WRITE_IOPS,
- resource_share);
- int64_t hdd_read_mbps = _get_resource_value(TResourceType::type::TRESOURCE_HDD_READ_MBPS,
- resource_share);
- int64_t hdd_write_mbps = _get_resource_value(TResourceType::type::TRESOURCE_HDD_WRITE_MBPS,
- resource_share);
- int64_t ssd_read_iops = _get_resource_value(TResourceType::type::TRESOURCE_SSD_READ_IOPS,
- resource_share);
- int64_t ssd_write_iops = _get_resource_value(TResourceType::type::TRESOURCE_SSD_WRITE_IOPS,
- resource_share);
- int64_t ssd_read_mbps = _get_resource_value(TResourceType::type::TRESOURCE_SSD_READ_MBPS,
- resource_share);
- int64_t ssd_write_mbps = _get_resource_value(TResourceType::type::TRESOURCE_SSD_WRITE_MBPS,
- resource_share);
-
- _config_disk_throttle(user_name, "", hdd_read_iops, hdd_write_iops,
- hdd_read_mbps, hdd_write_mbps,
- ssd_read_iops, ssd_write_iops,
- ssd_read_mbps, ssd_write_mbps);
- _config_disk_throttle(user_name, "low", hdd_read_iops, hdd_write_iops,
- hdd_read_mbps, hdd_write_mbps,
- ssd_read_iops, ssd_write_iops,
- ssd_read_mbps, ssd_write_mbps);
- _config_disk_throttle(user_name, "normal", hdd_read_iops, hdd_write_iops,
- hdd_read_mbps, hdd_write_mbps,
- ssd_read_iops, ssd_write_iops,
- ssd_read_mbps, ssd_write_mbps);
- _config_disk_throttle(user_name, "high", hdd_read_iops, hdd_write_iops,
- hdd_read_mbps, hdd_write_mbps,
- ssd_read_iops, ssd_write_iops,
- ssd_read_mbps, ssd_write_mbps);
+void CgroupsMgr::_config_user_disk_throttle(
+ std::string user_name, const std::map<TResourceType::type, int32_t>& resource_share) {
+ int64_t hdd_read_iops =
+ _get_resource_value(TResourceType::type::TRESOURCE_HDD_READ_IOPS, resource_share);
+ int64_t hdd_write_iops =
+ _get_resource_value(TResourceType::type::TRESOURCE_HDD_WRITE_IOPS, resource_share);
+ int64_t hdd_read_mbps =
+ _get_resource_value(TResourceType::type::TRESOURCE_HDD_READ_MBPS, resource_share);
+ int64_t hdd_write_mbps =
+ _get_resource_value(TResourceType::type::TRESOURCE_HDD_WRITE_MBPS, resource_share);
+ int64_t ssd_read_iops =
+ _get_resource_value(TResourceType::type::TRESOURCE_SSD_READ_IOPS, resource_share);
+ int64_t ssd_write_iops =
+ _get_resource_value(TResourceType::type::TRESOURCE_SSD_WRITE_IOPS, resource_share);
+ int64_t ssd_read_mbps =
+ _get_resource_value(TResourceType::type::TRESOURCE_SSD_READ_MBPS, resource_share);
+ int64_t ssd_write_mbps =
+ _get_resource_value(TResourceType::type::TRESOURCE_SSD_WRITE_MBPS, resource_share);
+
+ _config_disk_throttle(user_name, "", hdd_read_iops, hdd_write_iops, hdd_read_mbps,
+ hdd_write_mbps, ssd_read_iops, ssd_write_iops, ssd_read_mbps,
+ ssd_write_mbps);
+ _config_disk_throttle(user_name, "low", hdd_read_iops, hdd_write_iops, hdd_read_mbps,
+ hdd_write_mbps, ssd_read_iops, ssd_write_iops, ssd_read_mbps,
+ ssd_write_mbps);
+ _config_disk_throttle(user_name, "normal", hdd_read_iops, hdd_write_iops, hdd_read_mbps,
+ hdd_write_mbps, ssd_read_iops, ssd_write_iops, ssd_read_mbps,
+ ssd_write_mbps);
+ _config_disk_throttle(user_name, "high", hdd_read_iops, hdd_write_iops, hdd_read_mbps,
+ hdd_write_mbps, ssd_read_iops, ssd_write_iops, ssd_read_mbps,
+ ssd_write_mbps);
}
-int64_t CgroupsMgr::_get_resource_value(const TResourceType::type resource_type,
- const std::map<TResourceType::type, int32_t>& resource_share) {
+int64_t CgroupsMgr::_get_resource_value(
+ const TResourceType::type resource_type,
+ const std::map<TResourceType::type, int32_t>& resource_share) {
int64_t resource_value = -1;
std::map<TResourceType::type, int32_t>::const_iterator it = resource_share.find(resource_type);
if (it != resource_share.end()) {
@@ -165,16 +161,11 @@ int64_t CgroupsMgr::_get_resource_value(const TResourceType::type resource_type,
return resource_value;
}
-AgentStatus CgroupsMgr::_config_disk_throttle(std::string user_name,
- std::string level,
- int64_t hdd_read_iops,
- int64_t hdd_write_iops,
- int64_t hdd_read_mbps,
- int64_t hdd_write_mbps,
- int64_t ssd_read_iops,
- int64_t ssd_write_iops,
- int64_t ssd_read_mbps,
- int64_t ssd_write_mbps) {
+AgentStatus CgroupsMgr::_config_disk_throttle(std::string user_name, std::string level,
+ int64_t hdd_read_iops, int64_t hdd_write_iops,
+ int64_t hdd_read_mbps, int64_t hdd_write_mbps,
+ int64_t ssd_read_iops, int64_t ssd_write_iops,
+ int64_t ssd_read_mbps, int64_t ssd_write_mbps) {
string cgroups_path = this->_root_cgroups_path + "/" + user_name + "/" + level;
string read_bps_path = cgroups_path + "/blkio.throttle.read_bps_device";
string write_bps_path = cgroups_path + "/blkio.throttle.write_bps_device";
@@ -215,33 +206,25 @@ AgentStatus CgroupsMgr::_config_disk_throttle(std::string user_name,
int minor_number = minor(file_stat.st_dev);
minor_number = (minor_number / 16) * 16;
if (read_iops != -1) {
- ctrl_cmd << major_number << ":"
- << minor_number << " "
- << read_iops;
+ ctrl_cmd << major_number << ":" << minor_number << " " << read_iops;
_echo_cmd_to_cgroup(ctrl_cmd, read_iops_path);
ctrl_cmd.clear();
ctrl_cmd.str(std::string());
}
if (write_iops != -1) {
- ctrl_cmd << major_number << ":"
- << minor_number << " "
- << write_iops;
+ ctrl_cmd << major_number << ":" << minor_number << " " << write_iops;
_echo_cmd_to_cgroup(ctrl_cmd, write_iops_path);
ctrl_cmd.clear();
ctrl_cmd.str(std::string());
}
if (read_mbps != -1) {
- ctrl_cmd << major_number << ":"
- << minor_number << " "
- << (read_mbps << 20);
+ ctrl_cmd << major_number << ":" << minor_number << " " << (read_mbps << 20);
_echo_cmd_to_cgroup(ctrl_cmd, read_bps_path);
ctrl_cmd.clear();
ctrl_cmd.str(std::string());
}
if (write_mbps != -1) {
- ctrl_cmd << major_number << ":"
- << minor_number << " "
- << (write_mbps << 20);
+ ctrl_cmd << major_number << ":" << minor_number << " " << (write_mbps << 20);
_echo_cmd_to_cgroup(ctrl_cmd, write_bps_path);
ctrl_cmd.clear();
ctrl_cmd.str(std::string());
@@ -264,43 +247,42 @@ AgentStatus CgroupsMgr::modify_user_cgroups(const string& user_name,
// Traverse the user resource share map to append share value to cgroup's file
for (map<string, int32_t>::const_iterator user_resource = user_share.begin();
- user_resource != user_share.end(); ++user_resource){
- string resource_file_name = user_resource->first;
- int32_t user_share_weight = user_resource->second;
+ user_resource != user_share.end(); ++user_resource) {
+ string resource_file_name = user_resource->first;
+ int32_t user_share_weight = user_resource->second;
+ // Append the share_weight value to the file
+ string user_resource_path = user_cgroups_path + "/" + resource_file_name;
+ std::ofstream user_cgroups(user_resource_path.c_str(), std::ios::out | std::ios::app);
+ if (!user_cgroups.is_open()) {
+ return AgentStatus::DORIS_ERROR;
+ }
+ user_cgroups << user_share_weight << std::endl;
+ user_cgroups.close();
+ LOG(INFO) << "Append " << user_share_weight << " to " << user_resource_path;
+ for (map<string, int32_t>::const_iterator level_resource = level_share.begin();
+ level_resource != level_share.end(); ++level_resource) {
+ // Append resource share to level shares
+ string level_name = level_resource->first;
+ int32_t level_share_weight = level_resource->second;
+ // Check if the level cgroups exist
+ string level_cgroups_path = user_cgroups_path + "/" + level_name;
+ if (!is_file_exist(level_cgroups_path.c_str())) {
+ if (!boost::filesystem::create_directory(level_cgroups_path)) {
+ return AgentStatus::DORIS_ERROR;
+ }
+ }
+
// Append the share_weight value to the file
- string user_resource_path = user_cgroups_path + "/" + resource_file_name;
- std::ofstream user_cgroups(user_resource_path.c_str(), std::ios::out | std::ios::app);
- if (!user_cgroups.is_open()) {
+ string level_resource_path = level_cgroups_path + "/" + resource_file_name;
+ std::ofstream level_cgroups(level_resource_path.c_str(), std::ios::out | std::ios::app);
+ if (!level_cgroups.is_open()) {
return AgentStatus::DORIS_ERROR;
}
- user_cgroups << user_share_weight << std::endl;
- user_cgroups.close();
- LOG(INFO) << "Append " << user_share_weight << " to " << user_resource_path;
- for (map<string, int32_t>::const_iterator level_resource = level_share.begin();
- level_resource != level_share.end(); ++level_resource){
- // Append resource share to level shares
- string level_name = level_resource->first;
- int32_t level_share_weight = level_resource->second;
- // Check if the level cgroups exist
- string level_cgroups_path = user_cgroups_path + "/" + level_name;
- if (!is_file_exist(level_cgroups_path.c_str())) {
- if (!boost::filesystem::create_directory(level_cgroups_path)) {
- return AgentStatus::DORIS_ERROR;
- }
- }
-
- // Append the share_weight value to the file
- string level_resource_path = level_cgroups_path + "/" + resource_file_name;
- std::ofstream level_cgroups(level_resource_path.c_str(),
- std::ios::out | std::ios::app);
- if (!level_cgroups.is_open()) {
- return AgentStatus::DORIS_ERROR;
- }
- level_cgroups << level_share_weight << std::endl;
- level_cgroups.close();
-
- LOG(INFO) << "Append " << level_share_weight << " to " << level_resource_path;
- }
+ level_cgroups << level_share_weight << std::endl;
+ level_cgroups.close();
+
+ LOG(INFO) << "Append " << level_share_weight << " to " << level_resource_path;
+ }
}
return AgentStatus::DORIS_SUCCESS;
}
@@ -308,47 +290,44 @@ AgentStatus CgroupsMgr::modify_user_cgroups(const string& user_name,
AgentStatus CgroupsMgr::init_cgroups() {
std::string root_cgroups_tasks_path = this->_root_cgroups_path + "/tasks";
// Check if the root cgroups exists
- if (is_directory(this->_root_cgroups_path.c_str())
- && is_file_exist(root_cgroups_tasks_path.c_str())) {
- // Check the folder's virtual filesystem to find whether it is a cgroup file system
+ if (is_directory(this->_root_cgroups_path.c_str()) &&
+ is_file_exist(root_cgroups_tasks_path.c_str())) {
+ // Check the folder's virtual filesystem to find whether it is a cgroup file system
#ifndef BE_TEST
- struct statfs fs_type;
- statfs(root_cgroups_tasks_path.c_str(), &fs_type);
- if (fs_type.f_type != CGROUP_SUPER_MAGIC) {
- LOG(ERROR) << _root_cgroups_path << " is not a cgroups file system.";
- _is_cgroups_init_success = false;
- return AgentStatus::DORIS_ERROR;
- }
+ struct statfs fs_type;
+ statfs(root_cgroups_tasks_path.c_str(), &fs_type);
+ if (fs_type.f_type != CGROUP_SUPER_MAGIC) {
+ LOG(ERROR) << _root_cgroups_path << " is not a cgroups file system.";
+ _is_cgroups_init_success = false;
+ return AgentStatus::DORIS_ERROR;
+ }
#endif
- // Check if current user have write permission to cgroup folder
- if (access(_root_cgroups_path.c_str(), W_OK) != 0) {
- LOG(ERROR) << "Doris does not have write permission to "
- << _root_cgroups_path;
- _is_cgroups_init_success = false;
- return AgentStatus::DORIS_ERROR;
- }
- // If root folder exists, then delete all subfolders under it
- boost::filesystem::directory_iterator item_begin(this->_root_cgroups_path);
- boost::filesystem::directory_iterator item_end;
- for (; item_begin != item_end; item_begin++) {
- if (is_directory(item_begin->path().string().c_str())) {
- // Delete the sub folder
- if (delete_user_cgroups(item_begin->path().filename().string())
- != AgentStatus::DORIS_SUCCESS) {
- LOG(ERROR) << "Could not clean subfolder "
- << item_begin->path().string();
- _is_cgroups_init_success = false;
- return AgentStatus::DORIS_ERROR;
- }
+ // Check if current user have write permission to cgroup folder
+ if (access(_root_cgroups_path.c_str(), W_OK) != 0) {
+ LOG(ERROR) << "Doris does not have write permission to " << _root_cgroups_path;
+ _is_cgroups_init_success = false;
+ return AgentStatus::DORIS_ERROR;
+ }
+ // If root folder exists, then delete all subfolders under it
+ boost::filesystem::directory_iterator item_begin(this->_root_cgroups_path);
+ boost::filesystem::directory_iterator item_end;
+ for (; item_begin != item_end; item_begin++) {
+ if (is_directory(item_begin->path().string().c_str())) {
+ // Delete the sub folder
+ if (delete_user_cgroups(item_begin->path().filename().string()) !=
+ AgentStatus::DORIS_SUCCESS) {
+ LOG(ERROR) << "Could not clean subfolder " << item_begin->path().string();
+ _is_cgroups_init_success = false;
+ return AgentStatus::DORIS_ERROR;
}
}
- LOG(INFO) << "Initialize doris cgroups successfully under folder "
- << _root_cgroups_path;
- _is_cgroups_init_success = true;
- return AgentStatus::DORIS_SUCCESS;
+ }
+ LOG(INFO) << "Initialize doris cgroups successfully under folder " << _root_cgroups_path;
+ _is_cgroups_init_success = true;
+ return AgentStatus::DORIS_SUCCESS;
} else {
LOG(WARNING) << "Could not find a valid cgroups path for resource isolation,"
- << "current value is " << _root_cgroups_path << ". ignore it.";
+ << "current value is " << _root_cgroups_path << ". ignore it.";
_is_cgroups_init_success = false;
return AgentStatus::DORIS_ERROR;
}
@@ -362,8 +341,7 @@ void CgroupsMgr::apply_cgroup(const string& user_name, const string& level) {
s_global_cg_mgr->assign_to_cgroups(user_name, level);
}
-AgentStatus CgroupsMgr::assign_to_cgroups(const string& user_name,
- const string& level) {
+AgentStatus CgroupsMgr::assign_to_cgroups(const string& user_name, const string& level) {
if (!_is_cgroups_init_success) {
return AgentStatus::DORIS_ERROR;
}
@@ -371,17 +349,15 @@ AgentStatus CgroupsMgr::assign_to_cgroups(const string& user_name,
return assign_thread_to_cgroups(tid, user_name, level);
}
-AgentStatus CgroupsMgr::assign_thread_to_cgroups(int64_t thread_id,
- const string& user_name,
+AgentStatus CgroupsMgr::assign_thread_to_cgroups(int64_t thread_id, const string& user_name,
const string& level) {
if (!_is_cgroups_init_success) {
return AgentStatus::DORIS_ERROR;
}
string tasks_path = _root_cgroups_path + "/" + user_name + "/" + level + "/tasks";
if (!is_file_exist(_root_cgroups_path + "/" + user_name)) {
- tasks_path = this->_root_cgroups_path + "/"
- + _default_user_name + "/"
- + _default_level + "/tasks";
+ tasks_path = this->_root_cgroups_path + "/" + _default_user_name + "/" + _default_level +
+ "/tasks";
} else if (!is_file_exist(_root_cgroups_path + "/" + user_name + "/" + level)) {
tasks_path = this->_root_cgroups_path + "/" + user_name + "/tasks";
}
@@ -429,17 +405,15 @@ AgentStatus CgroupsMgr::drop_cgroups(const string& deleted_cgroups_path) {
// If failed then there maybe exist active tasks under it and try to relocate them
// Currently, try 10 times to relocate and delete the cgroups.
int32_t i = 0;
- while (is_file_exist(deleted_cgroups_path)
- && rmdir(deleted_cgroups_path.c_str()) < 0
- && i < this->_drop_retry_times) {
+ while (is_file_exist(deleted_cgroups_path) && rmdir(deleted_cgroups_path.c_str()) < 0 &&
+ i < this->_drop_retry_times) {
this->relocate_tasks(deleted_cgroups_path, this->_root_cgroups_path);
++i;
#ifdef BE_TEST
boost::filesystem::remove_all(deleted_cgroups_path);
#endif
- if (i == this->_drop_retry_times){
- LOG(ERROR) << "drop cgroups under path: " << deleted_cgroups_path
- << " failed.";
+ if (i == this->_drop_retry_times) {
+ LOG(ERROR) << "drop cgroups under path: " << deleted_cgroups_path << " failed.";
return AgentStatus::DORIS_ERROR;
}
}
@@ -470,8 +444,7 @@ AgentStatus CgroupsMgr::relocate_tasks(const string& src_cgroups, const string&
}
void CgroupsMgr::_echo_cmd_to_cgroup(stringstream& ctrl_cmd, string& cgroups_path) {
- std::ofstream cgroups_stream(cgroups_path.c_str(),
- std::ios::out | std::ios::app);
+ std::ofstream cgroups_stream(cgroups_path.c_str(), std::ios::out | std::ios::app);
if (cgroups_stream.is_open()) {
cgroups_stream << ctrl_cmd.str() << std::endl;
cgroups_stream.close();
diff --git a/be/src/agent/cgroups_mgr.h b/be/src/agent/cgroups_mgr.h
index 1f2afcc..4c30f19 100644
--- a/be/src/agent/cgroups_mgr.h
+++ b/be/src/agent/cgroups_mgr.h
@@ -18,11 +18,13 @@
#ifndef DORIS_BE_SRC_AGENT_CGROUPS_MGR_H
#define DORIS_BE_SRC_AGENT_CGROUPS_MGR_H
+#include <sys/types.h>
+
#include <cstdint>
#include <map>
#include <mutex>
#include <string>
-#include <sys/types.h>
+
#include "agent/status.h"
#include "gen_cpp/MasterService_types.h"
@@ -40,11 +42,11 @@ public:
// Compare the old user resource and new user resource to find deleted user
// then delete nonexisting cgroups, create new user cgroups, update all user cgroups
- AgentStatus update_local_cgroups(const TFetchResourceResult& new_fetched_resource);
-
+ AgentStatus update_local_cgroups(const TFetchResourceResult& new_fetched_resource);
+
// Delete all existing cgroups under root path
AgentStatus init_cgroups();
-
+
// Modify cgroup resource shares under cgroups_root_path.
// Create related cgroups if it not exist.
//
@@ -53,34 +55,29 @@ public:
//
// user_share: a mapping for shares for different resource like (cpu.share, 100)
// mapping key is resource file name in cgroup; value is share weight
- //
+ //
// level_share: a mapping for shares for different levels under the user.
// mapping key is level name; value is level's share. Currently, different resource using the same share.
- AgentStatus modify_user_cgroups(const std::string& user_name,
- const std::map<std::string, int32_t>& user_share,
+ AgentStatus modify_user_cgroups(const std::string& user_name,
+ const std::map<std::string, int32_t>& user_share,
const std::map<std::string, int32_t>& level_share);
- static void apply_cgroup(const std::string& user_name,
- const std::string& level);
+ static void apply_cgroup(const std::string& user_name, const std::string& level);
- static void apply_system_cgroup() {
- apply_cgroup(_s_system_user, _s_system_group);
- }
+ static void apply_system_cgroup() { apply_cgroup(_s_system_user, _s_system_group); }
// Assign the thread calling this funciton to the cgroup identified by user name and level
//
// Input parameters:
// user_name&level: the user name and level used to find the cgroup
- AgentStatus assign_to_cgroups(const std::string& user_name,
- const std::string& level);
+ AgentStatus assign_to_cgroups(const std::string& user_name, const std::string& level);
// Assign the thread identified by thread id to the cgroup identified by user name and level
//
// Input parameters:
// thread_id: the unique id for the thread
// user_name&level: the user name and level used to find the cgroup
- AgentStatus assign_thread_to_cgroups(int64_t thread_id,
- const std::string& user_name,
+ AgentStatus assign_thread_to_cgroups(int64_t thread_id, const std::string& user_name,
const std::string& level);
// Delete the user's cgroups and its sub level cgroups using DropCgroups
@@ -88,7 +85,7 @@ public:
// user name: user name to be deleted
AgentStatus delete_user_cgroups(const std::string& user_name);
// Delete a cgroup
- // If there are active tasks in this cgroups, they will be relocated
+ // If there are active tasks in this cgroups, they will be relocated
// to root cgroups.
// If there are sub cgroups, it will return error.
// Input parameters:
@@ -102,19 +99,17 @@ public:
// dest_cgroups: absolute path for dest cgroups folder
AgentStatus relocate_tasks(const std::string& src_cgroups, const std::string& dest_cgroups);
- int64_t get_cgroups_version() {
- return _cur_version;
- }
+ int64_t get_cgroups_version() { return _cur_version; }
// set the disk throttle for the user by getting resource value from the map and echo it to the cgroups.
- // currently, both the user and groups under the user are set to the same value
+ // currently, both the user and groups under the user are set to the same value
// because throttle does not support hierachy.
// Input parameters:
// user_name: name for the user
// resource_share: resource value get from fe
- void _config_user_disk_throttle(std::string user_name,
+ void _config_user_disk_throttle(std::string user_name,
const std::map<TResourceType::type, int32_t>& resource_share);
-
+
// get user resource share value from the map
int64_t _get_resource_value(const TResourceType::type resource_type,
const std::map<TResourceType::type, int32_t>& resource_share);
@@ -130,23 +125,18 @@ public:
// ssd_write_iops: write iops number for ssd disk.
// ssd_read_mbps: read bps number for ssd disk, using mb not byte or kb.
// ssd_write_mbps: write bps number for ssd disk, using mb not byte or kb.
- AgentStatus _config_disk_throttle(std::string user_name,
- std::string level,
- int64_t hdd_read_iops,
- int64_t hdd_write_iops,
- int64_t hdd_read_mbps,
- int64_t hdd_write_mbps,
- int64_t ssd_read_iops,
- int64_t ssd_write_iops,
- int64_t ssd_read_mbps,
- int64_t ssd_write_mbps);
-
+ AgentStatus _config_disk_throttle(std::string user_name, std::string level,
+ int64_t hdd_read_iops, int64_t hdd_write_iops,
+ int64_t hdd_read_mbps, int64_t hdd_write_mbps,
+ int64_t ssd_read_iops, int64_t ssd_write_iops,
+ int64_t ssd_read_mbps, int64_t ssd_write_mbps);
+
// echo command in string stream to the cgroup file
// Input parameters:
// ctrl_cmd: stringstream that contains the string to echo
// cgroups_path: target cgroup file path
void _echo_cmd_to_cgroup(std::stringstream& ctrl_cmd, std::string& cgroups_path);
-
+
// check if the path exists and it is a directory
// Input parameters:
// file_path: path to the file
@@ -165,6 +155,7 @@ public:
public:
const static std::string _s_system_user;
const static std::string _s_system_group;
+
private:
ExecEnv* _exec_env;
std::string _root_cgroups_path;
@@ -173,11 +164,11 @@ private:
std::string _default_user_name = "default";
std::string _default_level = "normal";
int64_t _cur_version;
- std::set<std::string> _local_users;
- std::mutex _update_cgroups_mtx;
+ std::set<std::string> _local_users;
+ std::mutex _update_cgroups_mtx;
// A static mapping from fe's resource type to cgroups file
- static std::map<TResourceType::type, std::string> _s_resource_cgroups;
+ static std::map<TResourceType::type, std::string> _s_resource_cgroups;
};
-}
+} // namespace doris
#endif
diff --git a/be/src/agent/heartbeat_server.cpp b/be/src/agent/heartbeat_server.cpp
index e17f272..47f12b5 100644
--- a/be/src/agent/heartbeat_server.cpp
+++ b/be/src/agent/heartbeat_server.cpp
@@ -16,22 +16,23 @@
// under the License.
#include "agent/heartbeat_server.h"
-#include <ctime>
-#include <fstream>
-#include <boost/filesystem.hpp>
#include <thrift/TProcessor.h>
+#include <boost/filesystem.hpp>
+#include <ctime>
+#include <fstream>
+
#include "common/status.h"
#include "gen_cpp/HeartbeatService.h"
#include "gen_cpp/Status_types.h"
#include "olap/storage_engine.h"
#include "olap/utils.h"
+#include "runtime/heartbeat_flags.h"
#include "service/backend_options.h"
#include "util/debug_util.h"
#include "util/thrift_server.h"
#include "util/time.h"
-#include "runtime/heartbeat_flags.h"
using std::fstream;
using std::nothrow;
@@ -41,9 +42,8 @@ using apache::thrift::transport::TProcessor;
namespace doris {
-HeartbeatServer::HeartbeatServer(TMasterInfo* master_info) :
- _master_info(master_info),
- _fe_epoch(0) {
+HeartbeatServer::HeartbeatServer(TMasterInfo* master_info)
+ : _master_info(master_info), _fe_epoch(0) {
_olap_engine = StorageEngine::instance();
_be_epoch = GetCurrentTimeMicros() / 1000;
}
@@ -52,16 +52,14 @@ void HeartbeatServer::init_cluster_id() {
_master_info->cluster_id = _olap_engine->effective_cluster_id();
}
-void HeartbeatServer::heartbeat(
- THeartbeatResult& heartbeat_result,
- const TMasterInfo& master_info) {
-
+void HeartbeatServer::heartbeat(THeartbeatResult& heartbeat_result,
+ const TMasterInfo& master_info) {
//print heartbeat in every minute
LOG_EVERY_N(INFO, 12) << "get heartbeat from FE."
- << "host:" << master_info.network_address.hostname
- << ", port:" << master_info.network_address.port
- << ", cluster id:" << master_info.cluster_id
- << ", counter:" << google::COUNTER;
+ << "host:" << master_info.network_address.hostname
+ << ", port:" << master_info.network_address.port
+ << ", cluster id:" << master_info.cluster_id
+ << ", counter:" << google::COUNTER;
// do heartbeat
Status st = _heartbeat(master_info);
@@ -83,7 +81,7 @@ Status HeartbeatServer::_heartbeat(const TMasterInfo& master_info) {
if (master_info.__isset.backend_ip) {
if (master_info.backend_ip != BackendOptions::get_localhost()) {
LOG(WARNING) << "backend ip saved in master does not equal to backend local ip"
- << master_info.backend_ip << " vs. " << BackendOptions::get_localhost();
+ << master_info.backend_ip << " vs. " << BackendOptions::get_localhost();
std::stringstream ss;
ss << "actual backend local ip: " << BackendOptions::get_localhost();
return Status::InternalError(ss.str());
@@ -101,7 +99,8 @@ Status HeartbeatServer::_heartbeat(const TMasterInfo& master_info) {
} else {
_master_info->cluster_id = master_info.cluster_id;
LOG(INFO) << "record cluster id. host: " << master_info.network_address.hostname
- << ". port: " << master_info.network_address.port << ". cluster id: " << master_info.cluster_id;
+ << ". port: " << master_info.network_address.port
+ << ". cluster id: " << master_info.cluster_id;
}
} else {
if (_master_info->cluster_id != master_info.cluster_id) {
@@ -111,19 +110,21 @@ Status HeartbeatServer::_heartbeat(const TMasterInfo& master_info) {
}
bool need_report = false;
- if (_master_info->network_address.hostname != master_info.network_address.hostname
- || _master_info->network_address.port != master_info.network_address.port) {
+ if (_master_info->network_address.hostname != master_info.network_address.hostname ||
+ _master_info->network_address.port != master_info.network_address.port) {
if (master_info.epoch > _fe_epoch) {
_master_info->network_address.hostname = master_info.network_address.hostname;
_master_info->network_address.port = master_info.network_address.port;
_fe_epoch = master_info.epoch;
need_report = true;
- LOG(INFO) << "master change. new master host: " << _master_info->network_address.hostname
- << ". port: " << _master_info->network_address.port << ". epoch: " << _fe_epoch;
+ LOG(INFO) << "master change. new master host: "
+ << _master_info->network_address.hostname
+ << ". port: " << _master_info->network_address.port
+ << ". epoch: " << _fe_epoch;
} else {
LOG(WARNING) << "epoch is not greater than local. ignore heartbeat. host: "
<< _master_info->network_address.hostname
- << " port: " << _master_info->network_address.port
+ << " port: " << _master_info->network_address.port
<< " local epoch: " << _fe_epoch
<< " received epoch: " << master_info.epoch;
return Status::InternalError("epoch is not greater than local. ignore heartbeat.");
@@ -169,12 +170,9 @@ Status HeartbeatServer::_heartbeat(const TMasterInfo& master_info) {
return Status::OK();
}
-AgentStatus create_heartbeat_server(
- ExecEnv* exec_env,
- uint32_t server_port,
- ThriftServer** thrift_server,
- uint32_t worker_thread_num,
- TMasterInfo* local_master_info) {
+AgentStatus create_heartbeat_server(ExecEnv* exec_env, uint32_t server_port,
+ ThriftServer** thrift_server, uint32_t worker_thread_num,
+ TMasterInfo* local_master_info) {
HeartbeatServer* heartbeat_server = new (nothrow) HeartbeatServer(local_master_info);
if (heartbeat_server == NULL) {
return DORIS_ERROR;
@@ -185,11 +183,8 @@ AgentStatus create_heartbeat_server(
boost::shared_ptr<HeartbeatServer> handler(heartbeat_server);
boost::shared_ptr<TProcessor> server_processor(new HeartbeatServiceProcessor(handler));
string server_name("heartbeat");
- *thrift_server = new ThriftServer(
- server_name,
- server_processor,
- server_port,
- worker_thread_num);
+ *thrift_server =
+ new ThriftServer(server_name, server_processor, server_port, worker_thread_num);
return DORIS_SUCCESS;
}
-} // namesapce doris
+} // namespace doris
diff --git a/be/src/agent/heartbeat_server.h b/be/src/agent/heartbeat_server.h
index 67816e4..5e84578 100644
--- a/be/src/agent/heartbeat_server.h
+++ b/be/src/agent/heartbeat_server.h
@@ -20,13 +20,12 @@
#include <mutex>
-#include "thrift/transport/TTransportUtils.h"
-
#include "agent/status.h"
#include "gen_cpp/HeartbeatService.h"
#include "gen_cpp/Status_types.h"
#include "olap/olap_define.h"
#include "runtime/exec_env.h"
+#include "thrift/transport/TTransportUtils.h"
namespace doris {
@@ -38,7 +37,7 @@ class ThriftServer;
class HeartbeatServer : public HeartbeatServiceIf {
public:
explicit HeartbeatServer(TMasterInfo* master_info);
- virtual ~HeartbeatServer() {};
+ virtual ~HeartbeatServer(){};
virtual void init_cluster_id();
@@ -64,13 +63,10 @@ private:
int64_t _fe_epoch;
DISALLOW_COPY_AND_ASSIGN(HeartbeatServer);
-}; // class HeartBeatServer
+}; // class HeartBeatServer
-AgentStatus create_heartbeat_server(
- ExecEnv* exec_env,
- uint32_t heartbeat_server_port,
- ThriftServer** heart_beat_server,
- uint32_t worker_thread_num,
- TMasterInfo* local_master_info);
-} // namespace doris
-#endif // DORIS_BE_SRC_AGENT_HEARTBEAT_SERVER_H
+AgentStatus create_heartbeat_server(ExecEnv* exec_env, uint32_t heartbeat_server_port,
+ ThriftServer** heart_beat_server, uint32_t worker_thread_num,
+ TMasterInfo* local_master_info);
+} // namespace doris
+#endif // DORIS_BE_SRC_AGENT_HEARTBEAT_SERVER_H
diff --git a/be/src/agent/pusher.cpp b/be/src/agent/pusher.cpp
index fa90b63..8e3ea72 100644
--- a/be/src/agent/pusher.cpp
+++ b/be/src/agent/pusher.cpp
@@ -16,16 +16,19 @@
// under the License.
#include "agent/pusher.h"
+
#include <pthread.h>
+
#include <cstdio>
#include <ctime>
#include <fstream>
#include <iostream>
#include <sstream>
#include <string>
+
+#include "agent/cgroups_mgr.h"
#include "boost/filesystem.hpp"
#include "boost/lexical_cast.hpp"
-#include "agent/cgroups_mgr.h"
#include "gen_cpp/AgentService_types.h"
#include "http/http_client.h"
#include "olap/olap_common.h"
@@ -40,24 +43,20 @@ using std::vector;
namespace doris {
-Pusher::Pusher(OLAPEngine* engine, const TPushReq& push_req) :
- _push_req(push_req), _engine(engine) {
-}
+Pusher::Pusher(OLAPEngine* engine, const TPushReq& push_req)
+ : _push_req(push_req), _engine(engine) {}
-Pusher::~Pusher() {
-}
+Pusher::~Pusher() {}
AgentStatus Pusher::init() {
AgentStatus status = DORIS_SUCCESS;
// Check replica exist
OLAPTablePtr olap_table;
- olap_table = _engine->get_table(
- _push_req.tablet_id,
- _push_req.schema_hash);
+ olap_table = _engine->get_table(_push_req.tablet_id, _push_req.schema_hash);
if (olap_table.get() == NULL) {
- OLAP_LOG_WARNING("get tables failed. tablet_id: %ld, schema_hash: %ld",
- _push_req.tablet_id, _push_req.schema_hash);
+ OLAP_LOG_WARNING("get tables failed. tablet_id: %ld, schema_hash: %ld", _push_req.tablet_id,
+ _push_req.schema_hash);
return DORIS_PUSH_INVALID_TABLE;
}
@@ -101,8 +100,8 @@ AgentStatus Pusher::_get_tmp_file_dir(const string& root_path, string* download_
if (0 != error_code) {
status = DORIS_ERROR;
- LOG(WARNING) << "create download dir failed.path: "
- << *download_path << ", error code: " << error_code;
+ LOG(WARNING) << "create download dir failed.path: " << *download_path
+ << ", error code: " << error_code;
}
}
@@ -130,13 +129,12 @@ AgentStatus Pusher::process(vector<TTabletInfo>* tablet_infos) {
estimate_time_out = config::download_low_speed_time;
}
bool is_timeout = false;
- auto download_cb = [this, estimate_time_out, file_size, &is_timeout] (HttpClient* client) {
+ auto download_cb = [this, estimate_time_out, file_size, &is_timeout](HttpClient* client) {
// Check timeout and set timeout
time_t now = time(NULL);
if (_push_req.timeout > 0 && _push_req.timeout < now) {
// return status to break this callback
- VLOG(3) << "check time out. time_out:" << _push_req.timeout
- << ", now:" << now;
+ VLOG(3) << "check time out. time_out:" << _push_req.timeout << ", now:" << now;
is_timeout = true;
return Status::OK();
}
@@ -159,7 +157,7 @@ AgentStatus Pusher::process(vector<TTabletInfo>* tablet_infos) {
uint64_t local_file_size = boost::filesystem::file_size(_local_file_path);
if (file_size != local_file_size) {
LOG(WARNING) << "download_file size error. file_size=" << file_size
- << ", local_file_size=" << local_file_size;
+ << ", local_file_size=" << local_file_size;
return Status::InternalError("downloaded file's size isn't right");
}
}
@@ -167,7 +165,7 @@ AgentStatus Pusher::process(vector<TTabletInfo>* tablet_infos) {
_push_req.http_file_path = _local_file_path;
return Status::OK();
};
-
+
MonotonicStopWatch stopwatch;
stopwatch.start();
auto st = HttpClient::execute_with_retry(MAX_RETRY, 1, download_cb);
@@ -178,18 +176,16 @@ AgentStatus Pusher::process(vector<TTabletInfo>* tablet_infos) {
if (st.ok() && !is_timeout) {
double rate = -1.0;
if (_push_req.__isset.http_file_size) {
- rate = (double) _push_req.http_file_size / (cost / 1000 / 1000 / 1000) / 1024;
+ rate = (double)_push_req.http_file_size / (cost / 1000 / 1000 / 1000) / 1024;
}
LOG(INFO) << "down load file success. local_file=" << _local_file_path
- << ", remote_file=" << _remote_file_path
- << ", tablet_id" << _push_req.tablet_id
- << ", cost=" << cost / 1000 << "us, file_size" << _push_req.http_file_size
- << ", download rage:" << rate << "KB/s";
+ << ", remote_file=" << _remote_file_path << ", tablet_id"
+ << _push_req.tablet_id << ", cost=" << cost / 1000 << "us, file_size"
+ << _push_req.http_file_size << ", download rage:" << rate << "KB/s";
} else {
LOG(WARNING) << "down load file failed. remote_file=" << _remote_file_path
- << ", tablet=" << _push_req.tablet_id
- << ", cost=" << cost / 1000
- << "us, errmsg=" << st.get_error_msg() << ", is_timeout=" << is_timeout;
+ << ", tablet=" << _push_req.tablet_id << ", cost=" << cost / 1000
+ << "us, errmsg=" << st.get_error_msg() << ", is_timeout=" << is_timeout;
status = DORIS_ERROR;
}
}
diff --git a/be/src/agent/status.h b/be/src/agent/status.h
index e5174ae..beea4c9 100644
--- a/be/src/agent/status.h
+++ b/be/src/agent/status.h
@@ -44,5 +44,5 @@ enum AgentStatus {
DORIS_INTERNAL_ERROR = -902,
DORIS_DISK_REACH_CAPACITY_LIMIT = -903
};
-} // namespace doris
-#endif // DORIS_BE_SRC_AGENT_STATUS_H
+} // namespace doris
+#endif // DORIS_BE_SRC_AGENT_STATUS_H
diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp
index fac22f0..e5c979c 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -329,8 +329,7 @@ void TaskWorkerPool::_create_tablet_worker_thread_callback() {
TStatus task_status;
std::vector<TTabletInfo> finish_tablet_infos;
- OLAPStatus create_status =
- _env->storage_engine()->create_tablet(create_tablet_req);
+ OLAPStatus create_status = _env->storage_engine()->create_tablet(create_tablet_req);
if (create_status != OLAPStatus::OLAP_SUCCESS) {
LOG(WARNING) << "create table failed. status: " << create_status
<< ", signature: " << agent_task_req.signature;
@@ -450,8 +449,7 @@ void TaskWorkerPool::_alter_tablet_worker_thread_callback() {
TTaskType::type task_type = agent_task_req.task_type;
switch (task_type) {
case TTaskType::ALTER:
- _alter_tablet(agent_task_req, signature,
- task_type, &finish_task_request);
+ _alter_tablet(agent_task_req, signature, task_type, &finish_task_request);
break;
default:
// pass
@@ -578,10 +576,9 @@ void TaskWorkerPool::_push_worker_thread_callback() {
return;
}
- index = _get_next_task_index(
- config::push_worker_count_normal_priority +
- config::push_worker_count_high_priority,
- _tasks, priority);
+ index = _get_next_task_index(config::push_worker_count_normal_priority +
+ config::push_worker_count_high_priority,
+ _tasks, priority);
if (index < 0) {
// there is no high priority task. notify other thread to handle normal task
@@ -626,8 +623,8 @@ void TaskWorkerPool::_push_worker_thread_callback() {
}
if (status == DORIS_SUCCESS) {
- VLOG(3) << "push ok. signature: " << agent_task_req.signature
- << ", push_type: " << push_req.push_type;
+ VLOG(3) << "push ok. signature: " << agent_task_req.signature
+ << ", push_type: " << push_req.push_type;
error_msgs.push_back("push success");
++_s_report_version;
@@ -867,8 +864,8 @@ void TaskWorkerPool::_clone_worker_thread_callback() {
std::vector<string> error_msgs;
std::vector<TTabletInfo> tablet_infos;
- EngineCloneTask engine_task(clone_req, _master_info,
- agent_task_req.signature, &error_msgs, &tablet_infos, &status);
+ EngineCloneTask engine_task(clone_req, _master_info, agent_task_req.signature, &error_msgs,
+ &tablet_infos, &status);
_env->storage_engine()->execute_task(&engine_task);
// Return result to fe
TStatus task_status;
@@ -919,18 +916,19 @@ void TaskWorkerPool::_storage_medium_migrate_worker_thread_callback() {
// check request and get info
TabletSharedPtr tablet;
DataDir* dest_store;
- if (_check_migrate_requset(storage_medium_migrate_req, tablet, &dest_store) != OLAP_SUCCESS) {
- status_code = TStatusCode::RUNTIME_ERROR;
+ if (_check_migrate_requset(storage_medium_migrate_req, tablet, &dest_store) !=
+ OLAP_SUCCESS) {
+ status_code = TStatusCode::RUNTIME_ERROR;
} else {
EngineStorageMigrationTask engine_task(tablet, dest_store);
OLAPStatus res = _env->storage_engine()->execute_task(&engine_task);
if (res != OLAP_SUCCESS) {
LOG(WARNING) << "storage media migrate failed. status: " << res
- << ", signature: " << agent_task_req.signature;
+ << ", signature: " << agent_task_req.signature;
status_code = TStatusCode::RUNTIME_ERROR;
} else {
LOG(INFO) << "storage media migrate success. status:" << res << ","
- << ", signature:" << agent_task_req.signature;
+ << ", signature:" << agent_task_req.signature;
}
}
@@ -950,17 +948,14 @@ void TaskWorkerPool::_storage_medium_migrate_worker_thread_callback() {
}
}
-OLAPStatus TaskWorkerPool::_check_migrate_requset(
- const TStorageMediumMigrateReq& req,
- TabletSharedPtr& tablet,
- DataDir** dest_store) {
-
+OLAPStatus TaskWorkerPool::_check_migrate_requset(const TStorageMediumMigrateReq& req,
+ TabletSharedPtr& tablet, DataDir** dest_store) {
int64_t tablet_id = req.tablet_id;
int32_t schema_hash = req.schema_hash;
tablet = StorageEngine::instance()->tablet_manager()->get_tablet(tablet_id, schema_hash);
if (tablet == nullptr) {
LOG(WARNING) << "can't find tablet. tablet_id= " << tablet_id
- << " schema_hash=" << schema_hash;
+ << " schema_hash=" << schema_hash;
return OLAP_ERR_TABLE_NOT_FOUND;
}
@@ -979,7 +974,7 @@ OLAPStatus TaskWorkerPool::_check_migrate_requset(
uint32_t count = StorageEngine::instance()->available_storage_medium_type_count();
if (count <= 1) {
LOG(INFO) << "available storage medium type count is less than 1, "
- << "no need to migrate. count=" << count;
+ << "no need to migrate. count=" << count;
return OLAP_REQUEST_FAILED;
}
// check current tablet storage medium
@@ -987,7 +982,7 @@ OLAPStatus TaskWorkerPool::_check_migrate_requset(
TStorageMedium::type src_storage_medium = tablet->data_dir()->storage_medium();
if (src_storage_medium == storage_medium) {
LOG(INFO) << "tablet is already on specified storage medium. "
- << "storage_medium=" << storage_medium;
+ << "storage_medium=" << storage_medium;
return OLAP_REQUEST_FAILED;
}
// get a random store of specified storage medium
@@ -1004,7 +999,7 @@ OLAPStatus TaskWorkerPool::_check_migrate_requset(
int64_t tablet_size = tablet->tablet_footprint();
if ((*dest_store)->reach_capacity_limit(tablet_size)) {
LOG(WARNING) << "reach the capacity limit of path: " << (*dest_store)->path()
- << ", tablet size: " << tablet_size;
+ << ", tablet size: " << tablet_size;
return OLAP_ERR_DISK_REACH_CAPACITY_LIMIT;
}
@@ -1080,15 +1075,16 @@ void TaskWorkerPool::_report_task_worker_thread_callback() {
if (status != DORIS_SUCCESS) {
DorisMetrics::instance()->report_task_requests_failed->increment(1);
- LOG(WARNING) << "report task failed. status: " << status << ", master host: "
- << _master_info.network_address.hostname
+ LOG(WARNING) << "report task failed. status: " << status
+ << ", master host: " << _master_info.network_address.hostname
<< "port: " << _master_info.network_address.port;
} else {
LOG(INFO) << "finish report task. master host: "
- << _master_info.network_address.hostname
- << " port: " << _master_info.network_address.port;
+ << _master_info.network_address.hostname
+ << " port: " << _master_info.network_address.port;
}
- } while (!_stop_background_threads_latch.wait_for(MonoDelta::FromSeconds(config::report_task_interval_seconds)));
+ } while (!_stop_background_threads_latch.wait_for(
+ MonoDelta::FromSeconds(config::report_task_interval_seconds)));
}
/// disk state report thread will report disk state at a configurable fix interval.
@@ -1108,7 +1104,8 @@ void TaskWorkerPool::_report_disk_state_worker_thread_callback() {
}
// wait at most report_disk_state_interval_seconds, or being notified
- _worker_thread_condition_variable.wait_for(MonoDelta::FromSeconds(config::report_disk_state_interval_seconds));
+ _worker_thread_condition_variable.wait_for(
+ MonoDelta::FromSeconds(config::report_disk_state_interval_seconds));
if (!_is_work) {
break;
}
@@ -1136,13 +1133,13 @@ void TaskWorkerPool::_report_disk_state_worker_thread_callback() {
if (status != DORIS_SUCCESS) {
DorisMetrics::instance()->report_disk_requests_failed->increment(1);
- LOG(WARNING) << "report disk state failed. status: " << status << ", master host: "
- << _master_info.network_address.hostname
+ LOG(WARNING) << "report disk state failed. status: " << status
+ << ", master host: " << _master_info.network_address.hostname
<< ", port: " << _master_info.network_address.port;
} else {
LOG(INFO) << "finish report disk state. master host: "
- << _master_info.network_address.hostname
- << ", port: " << _master_info.network_address.port;
+ << _master_info.network_address.hostname
+ << ", port: " << _master_info.network_address.port;
}
}
StorageEngine::instance()->deregister_report_listener(this);
@@ -1165,7 +1162,8 @@ void TaskWorkerPool::_report_tablet_worker_thread_callback() {
}
// wait at most report_tablet_interval_seconds, or being notified
- _worker_thread_condition_variable.wait_for(MonoDelta::FromSeconds(config::report_tablet_interval_seconds));
+ _worker_thread_condition_variable.wait_for(
+ MonoDelta::FromSeconds(config::report_tablet_interval_seconds));
if (!_is_work) {
break;
}
@@ -1194,8 +1192,8 @@ void TaskWorkerPool::_report_tablet_worker_thread_callback() {
<< ", port:" << _master_info.network_address.port;
} else {
LOG(INFO) << "finish report tablets. master host: "
- << _master_info.network_address.hostname
- << ", port: " << _master_info.network_address.port;
+ << _master_info.network_address.hostname
+ << ", port: " << _master_info.network_address.port;
}
}
StorageEngine::instance()->deregister_report_listener(this);
@@ -1223,8 +1221,7 @@ void TaskWorkerPool::_upload_worker_thread_callback() {
<< ", job id:" << upload_request.job_id;
std::map<int64_t, std::vector<std::string>> tablet_files;
- SnapshotLoader loader(_env, upload_request.job_id,
- agent_task_req.signature);
+ SnapshotLoader loader(_env, upload_request.job_id, agent_task_req.signature);
Status status = loader.upload(upload_request.src_dest_map, upload_request.broker_addr,
upload_request.broker_prop, &tablet_files);
@@ -1282,8 +1279,7 @@ void TaskWorkerPool::_download_worker_thread_callback() {
// TODO: download
std::vector<int64_t> downloaded_tablet_ids;
- SnapshotLoader loader(_env, download_request.job_id,
- agent_task_req.signature);
+ SnapshotLoader loader(_env, download_request.job_id, agent_task_req.signature);
Status status = loader.download(download_request.src_dest_map, download_request.broker_addr,
download_request.broker_prop, &downloaded_tablet_ids);
@@ -1486,9 +1482,9 @@ void TaskWorkerPool::_move_dir_thread_callback() {
TStatus task_status;
// TODO: move dir
- AgentStatus status = _move_dir(
- move_dir_req.tablet_id, move_dir_req.schema_hash, move_dir_req.src,
- move_dir_req.job_id, true /* TODO */, &error_msgs);
+ AgentStatus status =
+ _move_dir(move_dir_req.tablet_id, move_dir_req.schema_hash, move_dir_req.src,
+ move_dir_req.job_id, true /* TODO */, &error_msgs);
if (status != DORIS_SUCCESS) {
status_code = TStatusCode::RUNTIME_ERROR;
diff --git a/be/src/agent/task_worker_pool.h b/be/src/agent/task_worker_pool.h
index c40e6e1..fa62b06 100644
--- a/be/src/agent/task_worker_pool.h
+++ b/be/src/agent/task_worker_pool.h
@@ -32,8 +32,8 @@
#include "olap/olap_define.h"
#include "olap/storage_engine.h"
#include "util/condition_variable.h"
-#include "util/mutex.h"
#include "util/countdown_latch.h"
+#include "util/mutex.h"
#include "util/thread.h"
namespace doris {
@@ -73,38 +73,60 @@ public:
};
inline const std::string TYPE_STRING(TaskWorkerType type) {
- switch(type) {
- case CREATE_TABLE: return "CREATE_TABLE";
- case DROP_TABLE: return "DROP_TABLE";
- case PUSH: return "PUSH";
- case REALTIME_PUSH: return "REALTIME_PUSH";
- case PUBLISH_VERSION: return "PUBLISH_VERSION";
- case CLEAR_ALTER_TASK: return "CLEAR_ALTER_TASK";
- case CLEAR_TRANSACTION_TASK: return "CLEAR_TRANSACTION_TASK";
- case DELETE: return "DELETE";
- case ALTER_TABLE: return "ALTER_TABLE";
- case QUERY_SPLIT_KEY: return "QUERY_SPLIT_KEY";
- case CLONE: return "CLONE";
- case STORAGE_MEDIUM_MIGRATE: return "STORAGE_MEDIUM_MIGRATE";
- case CHECK_CONSISTENCY: return "CHECK_CONSISTENCY";
- case REPORT_TASK: return "REPORT_TASK";
- case REPORT_DISK_STATE: return "REPORT_DISK_STATE";
- case REPORT_OLAP_TABLE: return "REPORT_OLAP_TABLE";
- case UPLOAD: return "UPLOAD";
- case DOWNLOAD: return "DOWNLOAD";
- case MAKE_SNAPSHOT: return "MAKE_SNAPSHOT";
- case RELEASE_SNAPSHOT: return "RELEASE_SNAPSHOT";
- case MOVE: return "MOVE";
- case RECOVER_TABLET: return "RECOVER_TABLET";
- case UPDATE_TABLET_META_INFO: return "UPDATE_TABLET_META_INFO";
- default: return "Unknown";
+ switch (type) {
+ case CREATE_TABLE:
+ return "CREATE_TABLE";
+ case DROP_TABLE:
+ return "DROP_TABLE";
+ case PUSH:
+ return "PUSH";
+ case REALTIME_PUSH:
+ return "REALTIME_PUSH";
+ case PUBLISH_VERSION:
+ return "PUBLISH_VERSION";
+ case CLEAR_ALTER_TASK:
+ return "CLEAR_ALTER_TASK";
+ case CLEAR_TRANSACTION_TASK:
+ return "CLEAR_TRANSACTION_TASK";
+ case DELETE:
+ return "DELETE";
+ case ALTER_TABLE:
+ return "ALTER_TABLE";
+ case QUERY_SPLIT_KEY:
+ return "QUERY_SPLIT_KEY";
+ case CLONE:
+ return "CLONE";
+ case STORAGE_MEDIUM_MIGRATE:
+ return "STORAGE_MEDIUM_MIGRATE";
+ case CHECK_CONSISTENCY:
+ return "CHECK_CONSISTENCY";
+ case REPORT_TASK:
+ return "REPORT_TASK";
+ case REPORT_DISK_STATE:
+ return "REPORT_DISK_STATE";
+ case REPORT_OLAP_TABLE:
+ return "REPORT_OLAP_TABLE";
+ case UPLOAD:
+ return "UPLOAD";
+ case DOWNLOAD:
+ return "DOWNLOAD";
+ case MAKE_SNAPSHOT:
+ return "MAKE_SNAPSHOT";
+ case RELEASE_SNAPSHOT:
+ return "RELEASE_SNAPSHOT";
+ case MOVE:
+ return "MOVE";
+ case RECOVER_TABLET:
+ return "RECOVER_TABLET";
+ case UPDATE_TABLET_META_INFO:
+ return "UPDATE_TABLET_META_INFO";
+ default:
+ return "Unknown";
}
}
- TaskWorkerPool(
- const TaskWorkerType task_worker_type,
- ExecEnv* env,
- const TMasterInfo& master_info);
+ TaskWorkerPool(const TaskWorkerType task_worker_type, ExecEnv* env,
+ const TMasterInfo& master_info);
virtual ~TaskWorkerPool();
// Start the task worker thread pool
@@ -127,7 +149,7 @@ private:
void _remove_task_info(const TTaskType::type task_type, int64_t signature);
void _finish_task(const TFinishTaskRequest& finish_task_request);
uint32_t _get_next_task_index(int32_t thread_count, std::deque<TAgentTaskRequest>& tasks,
- TPriority::type priority);
+ TPriority::type priority);
void _create_tablet_worker_thread_callback();
void _drop_tablet_worker_thread_callback();
@@ -148,30 +170,18 @@ private:
void _move_dir_thread_callback();
void _update_tablet_meta_worker_thread_callback();
- void _alter_tablet(
- const TAgentTaskRequest& alter_tablet_request,
- int64_t signature,
- const TTaskType::type task_type,
- TFinishTaskRequest* finish_task_request);
-
- AgentStatus _get_tablet_info(
- const TTabletId tablet_id,
- const TSchemaHash schema_hash,
- int64_t signature,
- TTabletInfo* tablet_info);
-
- AgentStatus _move_dir(
- const TTabletId tablet_id,
- const TSchemaHash schema_hash,
- const std::string& src,
- int64_t job_id,
- bool overwrite,
- std::vector<std::string>* error_msgs);
-
- OLAPStatus _check_migrate_requset(
- const TStorageMediumMigrateReq& req,
- TabletSharedPtr& tablet,
- DataDir** dest_store);
+ void _alter_tablet(const TAgentTaskRequest& alter_tablet_request, int64_t signature,
+ const TTaskType::type task_type, TFinishTaskRequest* finish_task_request);
+
+ AgentStatus _get_tablet_info(const TTabletId tablet_id, const TSchemaHash schema_hash,
+ int64_t signature, TTabletInfo* tablet_info);
+
+ AgentStatus _move_dir(const TTabletId tablet_id, const TSchemaHash schema_hash,
+ const std::string& src, int64_t job_id, bool overwrite,
+ std::vector<std::string>* error_msgs);
+
+ OLAPStatus _check_migrate_requset(const TStorageMediumMigrateReq& req, TabletSharedPtr& tablet,
+ DataDir** dest_store);
private:
std::string _name;
@@ -201,6 +211,6 @@ private:
static std::map<TTaskType::type, std::set<int64_t>> _s_task_signatures;
DISALLOW_COPY_AND_ASSIGN(TaskWorkerPool);
-}; // class TaskWorkerPool
-} // namespace doris
-#endif // DORIS_BE_SRC_TASK_WORKER_POOL_H
+}; // class TaskWorkerPool
+} // namespace doris
+#endif // DORIS_BE_SRC_TASK_WORKER_POOL_H
diff --git a/be/src/agent/topic_listener.h b/be/src/agent/topic_listener.h
index 677be78..1d98c98 100644
--- a/be/src/agent/topic_listener.h
+++ b/be/src/agent/topic_listener.h
@@ -21,19 +21,17 @@
#include "gen_cpp/AgentService_types.h"
namespace doris {
-
-class TopicListener {
+class TopicListener {
public:
-
- virtual ~TopicListener(){}
+ virtual ~TopicListener() {}
// Deal with a single update
//
// Input parameters:
// protocol version: the version for the protocol, listeners should deal with the msg according to the protocol
// topic_update: single update
- virtual void handle_update(const TAgentServiceVersion::type& protocol_version,
+ virtual void handle_update(const TAgentServiceVersion::type& protocol_version,
const TTopicUpdate& topic_update) = 0;
};
-}
+} // namespace doris
#endif
diff --git a/be/src/agent/topic_subscriber.cpp b/be/src/agent/topic_subscriber.cpp
index 0e4c1fe..abc9b30 100644
--- a/be/src/agent/topic_subscriber.cpp
+++ b/be/src/agent/topic_subscriber.cpp
@@ -16,17 +16,17 @@
// under the License.
#include "agent/topic_subscriber.h"
+
#include "common/logging.h"
namespace doris {
-TopicSubscriber::TopicSubscriber() {
-}
+TopicSubscriber::TopicSubscriber() {}
TopicSubscriber::~TopicSubscriber() {
// Delete all listeners in the register
- std::map<TTopicType::type, std::vector<TopicListener*>>::iterator it
- = _registered_listeners.begin();
+ std::map<TTopicType::type, std::vector<TopicListener*>>::iterator it =
+ _registered_listeners.begin();
for (; it != _registered_listeners.end(); ++it) {
std::vector<TopicListener*>& listeners = it->second;
std::vector<TopicListener*>::iterator listener_it = listeners.begin();
@@ -53,9 +53,8 @@ void TopicSubscriber::handle_updates(const TAgentPublishRequest& agent_publish_r
std::vector<TopicListener*>::iterator listener_it = listeners.begin();
// Send the update to all listeners with protocol version.
for (; listener_it != listeners.end(); ++listener_it) {
- (*listener_it)->handle_update(agent_publish_request.protocol_version,
- *topic_update_it);
- }
+ (*listener_it)->handle_update(agent_publish_request.protocol_version, *topic_update_it);
+ }
}
}
} // namespace doris
diff --git a/be/src/agent/topic_subscriber.h b/be/src/agent/topic_subscriber.h
index cfde9a6..be71cda 100644
--- a/be/src/agent/topic_subscriber.h
+++ b/be/src/agent/topic_subscriber.h
@@ -18,17 +18,16 @@
#ifndef DORIS_BE_SRC_AGENT_TOPIC_SUBSCRIBER_H
#define DORIS_BE_SRC_AGENT_TOPIC_SUBSCRIBER_H
-#include <map>
#include <boost/thread.hpp>
+#include <map>
+
#include "agent/topic_listener.h"
#include "gen_cpp/AgentService_types.h"
namespace doris {
class TopicSubscriber {
-
public:
-
TopicSubscriber();
~TopicSubscriber();
// Put the topic type and listener to the map
diff --git a/be/src/agent/user_resource_listener.cpp b/be/src/agent/user_resource_listener.cpp
index 4c69f0b..2605772 100644
--- a/be/src/agent/user_resource_listener.cpp
+++ b/be/src/agent/user_resource_listener.cpp
@@ -16,13 +16,16 @@
// under the License.
#include "agent/user_resource_listener.h"
-#include <map>
-#include <future>
+
+#include <thrift/TApplicationException.h>
#include <thrift/Thrift.h>
-#include <thrift/transport/TSocket.h>
-#include <thrift/transport/TBufferTransports.h>
#include <thrift/protocol/TBinaryProtocol.h>
-#include <thrift/TApplicationException.h>
+#include <thrift/transport/TBufferTransports.h>
+#include <thrift/transport/TSocket.h>
+
+#include <future>
+#include <map>
+
#include "common/logging.h"
#include "gen_cpp/FrontendService.h"
#include "runtime/client_cache.h"
@@ -30,31 +33,27 @@
namespace doris {
using std::string;
-using apache::thrift::TException;
+using apache::thrift::TException;
using apache::thrift::transport::TTransportException;
// Initialize the resource to cgroups file mapping
// TRESOURCE_IOPS not mapped
-UserResourceListener::UserResourceListener(ExecEnv* exec_env,
- const TMasterInfo& master_info)
- : _master_info(master_info),
- _exec_env(exec_env),
- _cgroups_mgr(*(exec_env->cgroups_mgr())) {
-}
+UserResourceListener::UserResourceListener(ExecEnv* exec_env, const TMasterInfo& master_info)
+ : _master_info(master_info),
+ _exec_env(exec_env),
+ _cgroups_mgr(*(exec_env->cgroups_mgr())) {}
-UserResourceListener::~UserResourceListener() {
-}
+UserResourceListener::~UserResourceListener() {}
-void UserResourceListener::handle_update(const TAgentServiceVersion::type& protocol_version,
+void UserResourceListener::handle_update(const TAgentServiceVersion::type& protocol_version,
const TTopicUpdate& topic_update) {
- std::vector<TTopicItem> updates = topic_update.updates;
+ std::vector<TTopicItem> updates = topic_update.updates;
if (updates.size() > 0) {
int64_t new_version = updates[0].int_value;
// Async call to update users resource method
- std::async(std::launch::async,
- &UserResourceListener::update_users_resource,
- this, new_version);
+ std::async(std::launch::async, &UserResourceListener::update_users_resource, this,
+ new_version);
}
}
@@ -64,16 +63,15 @@ void UserResourceListener::update_users_resource(int64_t new_version) {
}
// Call fe to get latest user resource
Status master_status;
- // using 500ms as default timeout value
+ // using 500ms as default timeout value
FrontendServiceConnection client(_exec_env->frontend_client_cache(),
- _master_info.network_address,
- config::thrift_rpc_timeout_ms,
- &master_status);
+ _master_info.network_address, config::thrift_rpc_timeout_ms,
+ &master_status);
TFetchResourceResult new_fetched_resource;
- if (!master_status.ok()) {
- LOG(ERROR) << "Get frontend client failed, with address:"
- << _master_info.network_address.hostname << ":"
- << _master_info.network_address.port;
+ if (!master_status.ok()) {
+ LOG(ERROR) << "Get frontend client failed, with address:"
+ << _master_info.network_address.hostname << ":"
+ << _master_info.network_address.port;
return;
}
try {
@@ -83,24 +81,23 @@ void UserResourceListener::update_users_resource(int64_t new_version) {
// reopen the client and set timeout to 500ms
master_status = client.reopen(config::thrift_rpc_timeout_ms);
- if (!master_status.ok()) {
- LOG(WARNING) << "Reopen to get frontend client failed, with address:"
- << _master_info.network_address.hostname << ":"
- << _master_info.network_address.port;
+ if (!master_status.ok()) {
+ LOG(WARNING) << "Reopen to get frontend client failed, with address:"
+ << _master_info.network_address.hostname << ":"
+ << _master_info.network_address.port;
return;
}
LOG(WARNING) << "fetchResource from frontend failed, retry!";
client->fetchResource(new_fetched_resource);
}
- } catch (TException& e) {
+ } catch (TException& e) {
// Already try twice, log here
client.reopen(config::thrift_rpc_timeout_ms);
- LOG(WARNING) << "retry to fetchResource from "
- << _master_info.network_address.hostname << ":"
- << _master_info.network_address.port << " failed:\n"
- << e.what();
+ LOG(WARNING) << "retry to fetchResource from " << _master_info.network_address.hostname
+ << ":" << _master_info.network_address.port << " failed:\n"
+ << e.what();
return;
}
- _cgroups_mgr.update_local_cgroups(new_fetched_resource);
-}
+ _cgroups_mgr.update_local_cgroups(new_fetched_resource);
}
+} // namespace doris
diff --git a/be/src/agent/user_resource_listener.h b/be/src/agent/user_resource_listener.h
index 2fb6d97..31cb019 100644
--- a/be/src/agent/user_resource_listener.h
+++ b/be/src/agent/user_resource_listener.h
@@ -15,15 +15,16 @@
// specific language governing permissions and limitations
// under the License.
-#ifndef DORIS_BE_SRC_AGENT_USER_RESOURCE_LISTENER_H
+#ifndef DORIS_BE_SRC_AGENT_USER_RESOURCE_LISTENER_H
#define DORIS_BE_SRC_AGENT_USER_RESOURCE_LISTENER_H
#include <string>
-#include "agent/topic_listener.h"
+
#include "agent/cgroups_mgr.h"
+#include "agent/topic_listener.h"
#include "gen_cpp/AgentService_types.h"
-#include "gen_cpp/MasterService_types.h"
#include "gen_cpp/HeartbeatService_types.h"
+#include "gen_cpp/MasterService_types.h"
#include "runtime/exec_env.h"
namespace doris {
@@ -31,22 +32,22 @@ namespace doris {
class ExecEnv;
class UserResourceListener : public TopicListener {
-
public:
~UserResourceListener();
// Input parameters:
// root_cgroups_path: root cgroups allocated by admin to doris
UserResourceListener(ExecEnv* exec_env, const TMasterInfo& master_info);
// This method should be async
- virtual void handle_update(const TAgentServiceVersion::type& protocol_version,
+ virtual void handle_update(const TAgentServiceVersion::type& protocol_version,
const TTopicUpdate& topic_update);
+
private:
const TMasterInfo& _master_info;
ExecEnv* _exec_env;
- CgroupsMgr& _cgroups_mgr;
+ CgroupsMgr& _cgroups_mgr;
// Call cgroups mgr to update user's cgroups resource share
// Also refresh local user resource's cache
void update_users_resource(int64_t new_version);
-};
-}
+};
+} // namespace doris
#endif
diff --git a/be/src/agent/utils.cpp b/be/src/agent/utils.cpp
index a24effa..4ad4de2 100644
--- a/be/src/agent/utils.cpp
+++ b/be/src/agent/utils.cpp
@@ -17,15 +17,15 @@
#include "agent/utils.h"
-#include <cstdio>
-#include <fstream>
-#include <sstream>
-
#include <rapidjson/document.h>
#include <rapidjson/rapidjson.h>
#include <rapidjson/stringbuffer.h>
#include <rapidjson/writer.h>
+#include <cstdio>
+#include <fstream>
+#include <sstream>
+
#include "common/status.h"
using std::map;
@@ -37,22 +37,15 @@ using apache::thrift::transport::TTransportException;
namespace doris {
-MasterServerClient::MasterServerClient(
- const TMasterInfo& master_info,
- FrontendServiceClientCache* client_cache) :
- _master_info(master_info),
- _client_cache(client_cache) {
-}
+MasterServerClient::MasterServerClient(const TMasterInfo& master_info,
+ FrontendServiceClientCache* client_cache)
+ : _master_info(master_info), _client_cache(client_cache) {}
-AgentStatus MasterServerClient::finish_task(
- const TFinishTaskRequest& request,
- TMasterResult* result) {
+AgentStatus MasterServerClient::finish_task(const TFinishTaskRequest& request,
+ TMasterResult* result) {
Status client_status;
- FrontendServiceConnection client(
- _client_cache,
- _master_info.network_address,
- config::thrift_rpc_timeout_ms,
- &client_status);
+ FrontendServiceConnection client(_client_cache, _master_info.network_address,
+ config::thrift_rpc_timeout_ms, &client_status);
if (!client_status.ok()) {
LOG(WARNING) << "fail to get master client from cache. "
@@ -81,8 +74,7 @@ AgentStatus MasterServerClient::finish_task(
client.reopen(config::thrift_rpc_timeout_ms);
LOG(WARNING) << "fail to finish_task. "
<< "host=" << _master_info.network_address.hostname
- << ", port=" << _master_info.network_address.port
- << ", error=" << e.what();
+ << ", port=" << _master_info.network_address.port << ", error=" << e.what();
return DORIS_ERROR;
}
@@ -91,11 +83,8 @@ AgentStatus MasterServerClient::finish_task(
AgentStatus MasterServerClient::report(const TReportRequest& request, TMasterResult* result) {
Status client_status;
- FrontendServiceConnection client(
- _client_cache,
- _master_info.network_address,
- config::thrift_rpc_timeout_ms,
- &client_status);
+ FrontendServiceConnection client(_client_cache, _master_info.network_address,
+ config::thrift_rpc_timeout_ms, &client_status);
if (!client_status.ok()) {
LOG(WARNING) << "fail to get master client from cache. "
@@ -143,13 +132,11 @@ AgentStatus MasterServerClient::report(const TReportRequest& request, TMasterRes
return DORIS_SUCCESS;
}
-AgentStatus AgentUtils::rsync_from_remote(
- const string& remote_host,
- const string& remote_file_path,
- const string& local_file_path,
- const std::vector<string>& exclude_file_patterns,
- uint32_t transport_speed_limit_kbps,
- uint32_t timeout_second) {
+AgentStatus AgentUtils::rsync_from_remote(const string& remote_host, const string& remote_file_path,
+ const string& local_file_path,
+ const std::vector<string>& exclude_file_patterns,
+ uint32_t transport_speed_limit_kbps,
+ uint32_t timeout_second) {
int ret_code = 0;
std::stringstream cmd_stream;
cmd_stream << "rsync -r -q -e \"ssh -o StrictHostKeyChecking=no\"";
@@ -238,7 +225,7 @@ bool AgentUtils::exec_cmd(const string& command, string* errmsg, bool redirect_s
}
// Execute command.
- FILE *fp = popen(cmd.c_str(), "r");
+ FILE* fp = popen(cmd.c_str(), "r");
if (fp == NULL) {
std::stringstream err_stream;
err_stream << "popen failed. " << strerror(errno) << ", with errno: " << errno << ".\n";
@@ -259,8 +246,8 @@ bool AgentUtils::exec_cmd(const string& command, string* errmsg, bool redirect_s
*errmsg += "pclose cannot obtain the child status.\n";
} else {
std::stringstream err_stream;
- err_stream << "Close popen failed. " << strerror(errno) << ", with errno: "
- << errno << "\n";
+ err_stream << "Close popen failed. " << strerror(errno) << ", with errno: " << errno
+ << "\n";
*errmsg += err_stream.str();
}
return false;
@@ -269,7 +256,7 @@ bool AgentUtils::exec_cmd(const string& command, string* errmsg, bool redirect_s
// Get return code of command.
int32_t status_child = WEXITSTATUS(rc);
if (status_child == 0) {
- return true;
+ return true;
} else {
return false;
}
@@ -277,11 +264,10 @@ bool AgentUtils::exec_cmd(const string& command, string* errmsg, bool redirect_s
bool AgentUtils::write_json_to_file(const map<string, string>& info, const string& path) {
rapidjson::Document json_info(rapidjson::kObjectType);
- for (auto &it : info) {
- json_info.AddMember(
- rapidjson::Value(it.first.c_str(), json_info.GetAllocator()).Move(),
- rapidjson::Value(it.second.c_str(), json_info.GetAllocator()).Move(),
- json_info.GetAllocator());
+ for (auto& it : info) {
+ json_info.AddMember(rapidjson::Value(it.first.c_str(), json_info.GetAllocator()).Move(),
+ rapidjson::Value(it.second.c_str(), json_info.GetAllocator()).Move(),
+ json_info.GetAllocator());
}
rapidjson::StringBuffer json_info_str;
rapidjson::Writer<rapidjson::StringBuffer> writer(json_info_str);
@@ -296,4 +282,4 @@ bool AgentUtils::write_json_to_file(const map<string, string>& info, const strin
return true;
}
-} // namespace doris
+} // namespace doris
diff --git a/be/src/agent/utils.h b/be/src/agent/utils.h
index c732e03..1f40ec3 100644
--- a/be/src/agent/utils.h
+++ b/be/src/agent/utils.h
@@ -29,7 +29,7 @@ namespace doris {
class MasterServerClient {
public:
MasterServerClient(const TMasterInfo& master_info, FrontendServiceClientCache* client_cache);
- virtual ~MasterServerClient() {};
+ virtual ~MasterServerClient(){};
// Report finished task to the master server
//
@@ -59,8 +59,8 @@ private:
class AgentUtils {
public:
- AgentUtils() {};
- virtual ~AgentUtils() {};
+ AgentUtils(){};
+ virtual ~AgentUtils(){};
// Use rsync synchronize folder from remote agent to local folder
//
@@ -71,28 +71,27 @@ public:
// * exclude_file_patterns: the patterns of the exclude file
// * transport_speed_limit_kbps: speed limit of transport(kb/s)
// * timeout_second: timeout of synchronize
- virtual AgentStatus rsync_from_remote(
- const std::string& remote_host,
- const std::string& remote_file_path,
- const std::string& local_file_path,
- const std::vector<std::string>& exclude_file_patterns,
- const uint32_t transport_speed_limit_kbps,
- const uint32_t timeout_second);
+ virtual AgentStatus rsync_from_remote(const std::string& remote_host,
+ const std::string& remote_file_path,
+ const std::string& local_file_path,
+ const std::vector<std::string>& exclude_file_patterns,
+ const uint32_t transport_speed_limit_kbps,
+ const uint32_t timeout_second);
// Print AgentStatus as string
virtual std::string print_agent_status(AgentStatus status);
// Execute shell cmd
- virtual bool exec_cmd(const std::string& command, std::string* errmsg, bool redirect_stderr = true);
+ virtual bool exec_cmd(const std::string& command, std::string* errmsg,
+ bool redirect_stderr = true);
// Write a map to file by json format
- virtual bool write_json_to_file(
- const std::map<std::string, std::string>& info,
- const std::string& path);
+ virtual bool write_json_to_file(const std::map<std::string, std::string>& info,
+ const std::string& path);
private:
DISALLOW_COPY_AND_ASSIGN(AgentUtils);
-}; // class AgentUtils
+}; // class AgentUtils
-} // namespace doris
-#endif // DORIS_BE_SRC_AGENT_UTILS_H
+} // namespace doris
+#endif // DORIS_BE_SRC_AGENT_UTILS_H
diff --git a/be/src/common/atomic.h b/be/src/common/atomic.h
index c270411..f9a6416 100644
--- a/be/src/common/atomic.h
+++ b/be/src/common/atomic.h
@@ -36,29 +36,25 @@ public:
// while (1) CpuWait();
static ALWAYS_INLINE void cpu_wait() {
#if (defined(__i386) || defined(__x86_64__))
- asm volatile("pause\n": : :"memory");
+ asm volatile("pause\n" : : : "memory");
#elif defined(__aarch64__)
asm volatile("yield\n" ::: "memory");
#endif
}
/// Provides "barrier" semantics (see below) without a memory access.
- static ALWAYS_INLINE void memory_barrier() {
- __sync_synchronize();
- }
+ static ALWAYS_INLINE void memory_barrier() { __sync_synchronize(); }
/// Provides a compiler barrier. The compiler is not allowed to reorder memory
/// accesses across this (but the CPU can). This generates no instructions.
- static ALWAYS_INLINE void compiler_barrier() {
- __asm__ __volatile__("" : : : "memory");
- }
+ static ALWAYS_INLINE void compiler_barrier() { __asm__ __volatile__("" : : : "memory"); }
};
// Wrapper for atomic integers. This should be switched to c++ 11 when
// we can switch.
// This class overloads operators to behave like a regular integer type
// but all operators and functions are thread safe.
-template<typename T>
+template <typename T>
class AtomicInt {
public:
AtomicInt(T initial) : _value(initial) {}
@@ -114,34 +110,22 @@ public:
}
// Safe read of the value
- T read() {
- return __sync_fetch_and_add(&_value, 0);
- }
+ T read() { return __sync_fetch_and_add(&_value, 0); }
/// Atomic load with "acquire" memory-ordering semantic.
- ALWAYS_INLINE T load() const {
- return base::subtle::Acquire_Load(&_value);
- }
+ ALWAYS_INLINE T load() const { return base::subtle::Acquire_Load(&_value); }
/// Atomic store with "release" memory-ordering semantic.
- ALWAYS_INLINE void store(T x) {
- base::subtle::Release_Store(&_value, x);
- }
+ ALWAYS_INLINE void store(T x) { base::subtle::Release_Store(&_value, x); }
/// Atomic add with "barrier" memory-ordering semantic. Returns the new value.
- ALWAYS_INLINE T add(T x) {
- return base::subtle::Barrier_AtomicIncrement(&_value, x);
- }
+ ALWAYS_INLINE T add(T x) { return base::subtle::Barrier_AtomicIncrement(&_value, x); }
// Increments by delta (i.e. += delta) and returns the new val
- T update_and_fetch(T delta) {
- return __sync_add_and_fetch(&_value, delta);
- }
+ T update_and_fetch(T delta) { return __sync_add_and_fetch(&_value, delta); }
// Increment by delta and returns the old val
- T fetch_and_update(T delta) {
- return __sync_fetch_and_add(&_value, delta);
- }
+ T fetch_and_update(T delta) { return __sync_fetch_and_add(&_value, delta); }
// Updates the int to 'value' if value is larger
void update_max(T value) {
@@ -176,9 +160,7 @@ public:
}
// Atomically updates _value with new_val. Returns the old _value.
- T swap(const T& new_val) {
- return __sync_lock_test_and_set(&_value, new_val);
- }
+ T swap(const T& new_val) { return __sync_lock_test_and_set(&_value, new_val); }
private:
T _value;
@@ -190,7 +172,7 @@ typedef AtomicInt<int32_t> AtomicInt32;
typedef AtomicInt<int64_t> AtomicInt64;
/// Atomic pointer. Operations have the same semantics as AtomicInt.
-template<typename T>
+template <typename T>
class AtomicPtr {
public:
AtomicPtr(T* initial = nullptr) : _ptr(reinterpret_cast<intptr_t>(initial)) {}
@@ -204,7 +186,7 @@ public:
/// Store 'new_val' and return the previous value. Implies a Release memory barrier
/// (i.e. the same as Store()).
inline T* swap(T* val) {
- return reinterpret_cast<T*>(_ptr.swap(reinterpret_cast<intptr_t>(val)));
+ return reinterpret_cast<T*>(_ptr.swap(reinterpret_cast<intptr_t>(val)));
}
private:
diff --git a/be/src/common/compiler_util.h b/be/src/common/compiler_util.h
index e9e9e90..accc70c 100644
--- a/be/src/common/compiler_util.h
+++ b/be/src/common/compiler_util.h
@@ -44,7 +44,6 @@
/// decision, e.g. not inlining a small function on a hot path.
#define ALWAYS_INLINE __attribute__((always_inline))
-#define ALIGN_CACHE_LINE __attribute__ ((aligned (CACHE_LINE_SIZE)))
+#define ALIGN_CACHE_LINE __attribute__((aligned(CACHE_LINE_SIZE)))
#endif
-
diff --git a/be/src/common/config.h b/be/src/common/config.h
index e1fc535..75ed1eb 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -22,595 +22,594 @@
namespace doris {
namespace config {
- // Dir of custom config file
- CONF_String(custom_config_dir, "${DORIS_HOME}/conf");
-
- // cluster id
- CONF_Int32(cluster_id, "-1");
- // port on which BackendService is exported
- CONF_Int32(be_port, "9060");
-
- // port for brpc
- CONF_Int32(brpc_port, "8060");
-
- // the number of bthreads for brpc, the default value is set to -1, which means the number of bthreads is #cpu-cores
- CONF_Int32(brpc_num_threads, "-1")
-
- // Declare a selection strategy for those servers have many ips.
- // Note that there should at most one ip match this list.
- // this is a list in semicolon-delimited format, in CIDR notation, e.g. 10.10.10.0/24
- // If no ip match this rule, will choose one randomly.
- CONF_String(priority_networks, "");
-
- ////
- //// tcmalloc gc parameter
- ////
- // min memory for TCmalloc, when used memory is smaller than this, do not returned to OS
- CONF_mInt64(tc_use_memory_min, "10737418240");
- // free memory rate.[0-100]
- CONF_mInt64(tc_free_memory_rate, "20");
-
- // Bound on the total amount of bytes allocated to thread caches.
- // This bound is not strict, so it is possible for the cache to go over this bound
- // in certain circumstances. This value defaults to 1GB
- // If you suspect your application is not scaling to many threads due to lock contention in TCMalloc,
- // you can try increasing this value. This may improve performance, at a cost of extra memory
- // use by TCMalloc.
- // reference: https://gperftools.github.io/gperftools/tcmalloc.html: TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES
- // https://github.com/gperftools/gperftools/issues/1111
- CONF_Int64(tc_max_total_thread_cache_bytes, "1073741824");
-
- // process memory limit specified as number of bytes
- // ('<int>[bB]?'), megabytes ('<float>[mM]'), gigabytes ('<float>[gG]'),
- // or percentage of the physical memory ('<int>%').
- // defaults to bytes if no unit is given"
- // must larger than 0. and if larger than physical memory size,
- // it will be set to physical memory size.
- CONF_String(mem_limit, "80%");
-
- // the port heartbeat service used
- CONF_Int32(heartbeat_service_port, "9050");
- // the count of heart beat service
- CONF_Int32(heartbeat_service_thread_count, "1");
- // the count of thread to create table
- CONF_Int32(create_tablet_worker_count, "3");
- // the count of thread to drop table
- CONF_Int32(drop_tablet_worker_count, "3");
- // the count of thread to batch load
- CONF_Int32(push_worker_count_normal_priority, "3");
- // the count of thread to high priority batch load
- CONF_Int32(push_worker_count_high_priority, "3");
- // the count of thread to publish version
- CONF_Int32(publish_version_worker_count, "8");
- // the count of thread to clear transaction task
- CONF_Int32(clear_transaction_task_worker_count, "1");
- // the count of thread to delete
- CONF_Int32(delete_worker_count, "3");
- // the count of thread to alter table
- CONF_Int32(alter_tablet_worker_count, "3");
- // the count of thread to clone
- CONF_Int32(clone_worker_count, "3");
- // the count of thread to clone
- CONF_Int32(storage_medium_migrate_count, "1");
- // the count of thread to check consistency
- CONF_Int32(check_consistency_worker_count, "1");
- // the count of thread to upload
- CONF_Int32(upload_worker_count, "1");
- // the count of thread to download
- CONF_Int32(download_worker_count, "1");
- // the count of thread to make snapshot
- CONF_Int32(make_snapshot_worker_count, "5");
- // the count of thread to release snapshot
- CONF_Int32(release_snapshot_worker_count, "5");
- // the interval time(seconds) for agent report tasks signatrue to FE
- CONF_mInt32(report_task_interval_seconds, "10");
- // the interval time(seconds) for agent report disk state to FE
- CONF_mInt32(report_disk_state_interval_seconds, "60");
- // the interval time(seconds) for agent report olap table to FE
- CONF_mInt32(report_tablet_interval_seconds, "60");
- // the interval time(seconds) for agent report plugin status to FE
- // CONF_Int32(report_plugin_interval_seconds, "120");
- // the timeout(seconds) for alter table
- // CONF_Int32(alter_tablet_timeout_seconds, "86400");
- // the timeout(seconds) for make snapshot
- // CONF_Int32(make_snapshot_timeout_seconds, "600");
- // the timeout(seconds) for release snapshot
- // CONF_Int32(release_snapshot_timeout_seconds, "600");
- // the max download speed(KB/s)
- CONF_mInt32(max_download_speed_kbps, "50000");
- // download low speed limit(KB/s)
- CONF_mInt32(download_low_speed_limit_kbps, "50");
- // download low speed time(seconds)
- CONF_mInt32(download_low_speed_time, "300");
- // curl verbose mode
- // CONF_Int64(curl_verbose_mode, "1");
- // seconds to sleep for each time check table status
- // CONF_Int32(check_status_sleep_time_seconds, "10");
- // sleep time for one second
- CONF_Int32(sleep_one_second, "1");
- // sleep time for five seconds
- CONF_Int32(sleep_five_seconds, "5");
-
- // log dir
- CONF_String(sys_log_dir, "${DORIS_HOME}/log");
- CONF_String(user_function_dir, "${DORIS_HOME}/lib/udf");
- // INFO, WARNING, ERROR, FATAL
- CONF_String(sys_log_level, "INFO");
- // TIME-DAY, TIME-HOUR, SIZE-MB-nnn
- CONF_String(sys_log_roll_mode, "SIZE-MB-1024");
- // log roll num
- CONF_Int32(sys_log_roll_num, "10");
- // verbose log
- CONF_Strings(sys_log_verbose_modules, "");
- // verbose log level
- CONF_Int32(sys_log_verbose_level, "10");
- // log buffer level
- CONF_String(log_buffer_level, "");
-
- // Pull load task dir
- CONF_String(pull_load_task_dir, "${DORIS_HOME}/var/pull_load");
-
- // the maximum number of bytes to display on the debug webserver's log page
- CONF_Int64(web_log_bytes, "1048576");
- // number of threads available to serve backend execution requests
- CONF_Int32(be_service_threads, "64");
- // key=value pair of default query options for Doris, separated by ','
- CONF_String(default_query_options, "");
-
- // If non-zero, Doris will output memory usage every log_mem_usage_interval'th fragment completion.
- // CONF_Int32(log_mem_usage_interval, "0");
-
- // cgroups allocated for doris
- CONF_String(doris_cgroups, "");
-
- // Controls the number of threads to run work per core. It's common to pick 2x
- // or 3x the number of cores. This keeps the cores busy without causing excessive
- // thrashing.
- CONF_Int32(num_threads_per_core, "3");
- // if true, compresses tuple data in Serialize
- CONF_Bool(compress_rowbatches, "true");
- // serialize and deserialize each returned row batch
- CONF_Bool(serialize_batch, "false");
- // interval between profile reports; in seconds
- CONF_mInt32(status_report_interval, "5");
- // Local directory to copy UDF libraries from HDFS into
- CONF_String(local_library_dir, "${UDF_RUNTIME_DIR}");
- // number of olap scanner thread pool size
- CONF_Int32(doris_scanner_thread_pool_thread_num, "48");
- // number of olap scanner thread pool queue size
- CONF_Int32(doris_scanner_thread_pool_queue_size, "102400");
- // number of etl thread pool size
- CONF_Int32(etl_thread_pool_size, "8");
- // number of etl thread pool size
- CONF_Int32(etl_thread_pool_queue_size, "256");
- // port on which to run Doris test backend
- CONF_Int32(port, "20001");
- // default thrift client connect timeout(in seconds)
- CONF_Int32(thrift_connect_timeout_seconds, "3");
- // default thrift client retry interval (in milliseconds)
- CONF_mInt64(thrift_client_retry_interval_ms, "1000");
- // max row count number for single scan range
- CONF_mInt32(doris_scan_range_row_count, "524288");
- // size of scanner queue between scanner thread and compute thread
- CONF_mInt32(doris_scanner_queue_size, "1024");
- // single read execute fragment row size
- CONF_mInt32(doris_scanner_row_num, "16384");
- // number of max scan keys
- CONF_mInt32(doris_max_scan_key_num, "1024");
- // the max number of push down values of a single column.
- // if exceed, no conditions will be pushed down for that column.
- CONF_mInt32(max_pushdown_conditions_per_column, "1024");
- // return_row / total_row
- CONF_mInt32(doris_max_pushdown_conjuncts_return_rate, "90");
- // (Advanced) Maximum size of per-query receive-side buffer
- CONF_mInt32(exchg_node_buffer_size_bytes, "10485760");
- // insert sort threshold for sorter
- // CONF_Int32(insertion_threshold, "16");
- // the block_size every block allocate for sorter
- CONF_Int32(sorter_block_size, "8388608");
- // push_write_mbytes_per_sec
- CONF_Int32(push_write_mbytes_per_sec, "10");
-
- CONF_mInt64(column_dictionary_key_ratio_threshold, "0");
- CONF_mInt64(column_dictionary_key_size_threshold, "0");
- // if true, output IR after optimization passes
- // CONF_Bool(dump_ir, "false");
- // if set, saves the generated IR to the output file.
- //CONF_String(module_output, "");
- // memory_limitation_per_thread_for_schema_change unit GB
- CONF_mInt32(memory_limitation_per_thread_for_schema_change, "2");
-
- // CONF_Int64(max_unpacked_row_block_size, "104857600");
-
- CONF_mInt32(file_descriptor_cache_clean_interval, "3600");
- CONF_mInt32(disk_stat_monitor_interval, "5");
- CONF_mInt32(unused_rowset_monitor_interval, "30");
- CONF_String(storage_root_path, "${DORIS_HOME}/storage");
-
- // Config is used to check incompatible old format hdr_ format
- // whether doris uses strict way. When config is true, process will log fatal
- // and exit. When config is false, process will only log warning.
- CONF_Bool(storage_strict_check_incompatible_old_format, "true");
-
- // BE process will exit if the percentage of error disk reach this value.
- CONF_mInt32(max_percentage_of_error_disk, "0");
- // CONF_Int32(default_num_rows_per_data_block, "1024");
- CONF_mInt32(default_num_rows_per_column_file_block, "1024");
- // pending data policy
- CONF_mInt32(pending_data_expire_time_sec, "1800");
- // inc_rowset expired interval
- CONF_mInt32(inc_rowset_expired_sec, "1800");
- // inc_rowset snapshot rs sweep time interval
- CONF_mInt32(tablet_rowset_stale_sweep_time_sec, "1800");
- // garbage sweep policy
- CONF_Int32(max_garbage_sweep_interval, "3600");
- CONF_Int32(min_garbage_sweep_interval, "180");
- CONF_mInt32(snapshot_expire_time_sec, "172800");
- // 仅仅是建议值，当磁盘空间不足时，trash下的文件保存期可不遵守这个参数
- CONF_mInt32(trash_file_expire_time_sec, "259200");
- // check row nums for BE/CE and schema change. true is open, false is closed.
- CONF_mBool(row_nums_check, "true");
- //file descriptors cache, by default, cache 32768 descriptors
- CONF_Int32(file_descriptor_cache_capacity, "32768");
- // minimum file descriptor number
- // modify them upon necessity
- CONF_Int32(min_file_descriptor_number, "60000");
- CONF_Int64(index_stream_cache_capacity, "10737418240");
- // CONF_Int64(max_packed_row_block_size, "20971520");
-
- // Cache for storage page size
- CONF_String(storage_page_cache_limit, "20G");
- // whether to disable page cache feature in storage
- CONF_Bool(disable_storage_page_cache, "false");
-
- // be policy
- // whether disable automatic compaction task
- CONF_mBool(disable_auto_compaction, "false");
- // check the configuration of auto compaction in seconds when auto compaction disabled
- CONF_mInt32(check_auto_compaction_interval_seconds, "5");
-
- // CONF_Int64(base_compaction_start_hour, "20");
- // CONF_Int64(base_compaction_end_hour, "7");
- CONF_mInt64(base_compaction_num_cumulative_deltas, "5");
- CONF_mDouble(base_cumulative_delta_ratio, "0.3");
- CONF_mInt64(base_compaction_interval_seconds_since_last_operation, "86400");
- CONF_mInt32(base_compaction_write_mbytes_per_sec, "5");
-
- // config the cumulative compaction policy
- // Valid configs: num_base, size_based
- // num_based policy, the original version of cumulative compaction, cumulative version compaction once.
- // size_based policy, a optimization version of cumulative compaction, targeting the use cases requiring
- // lower write amplification, trading off read amplification and space amplification.
- CONF_String(cumulative_compaction_policy, "size_based");
-
- // In size_based policy, output rowset of cumulative compaction total disk size exceed this config size,
- // this rowset will be given to base compaction, unit is m byte.
- CONF_mInt64(cumulative_size_based_promotion_size_mbytes, "1024");
- // In size_based policy, output rowset of cumulative compaction total disk size exceed this config ratio of
- // base rowset's total disk size, this rowset will be given to base compaction. The value must be between
- // 0 and 1.
- CONF_mDouble(cumulative_size_based_promotion_ratio, "0.05");
- // In size_based policy, the smallest size of rowset promotion. When the rowset is less than this config, this
- // rowset will be not given to base compaction. The unit is m byte.
- CONF_mInt64(cumulative_size_based_promotion_min_size_mbytes, "64");
- // The lower bound size to do cumulative compaction. When total disk size of candidate rowsets is less than
- // this size, size_based policy may not do to cumulative compaction. The unit is m byte.
- CONF_mInt64(cumulative_size_based_compaction_lower_size_mbytes, "64");
-
- // cumulative compaction policy: min and max delta file's number
- CONF_mInt64(min_cumulative_compaction_num_singleton_deltas, "5");
- CONF_mInt64(max_cumulative_compaction_num_singleton_deltas, "1000");
- CONF_mInt64(cumulative_compaction_budgeted_bytes, "104857600");
- // CONF_Int32(cumulative_compaction_write_mbytes_per_sec, "100");
- // cumulative compaction skips recently published deltas in order to prevent
- // compacting a version that might be queried (in case the query planning phase took some time).
- // the following config set the window size
- CONF_mInt32(cumulative_compaction_skip_window_seconds, "30");
-
- // if compaction of a tablet failed, this tablet should not be chosen to
- // compaction until this interval passes.
- CONF_mInt64(min_compaction_failure_interval_sec, "600"); // 10 min
-
- // This config can be set to limit thread number in compaction thread pool.
- CONF_mInt32(min_compaction_threads, "10");
- CONF_mInt32(max_compaction_threads, "10");
-
- // The upper limit of "permits" held by all compaction tasks. This config can be set to limit memory consumption for compaction.
- CONF_mInt64(total_permits_for_compaction_score, "10000");
-
- // Compaction task number per disk.
- CONF_mInt32(compaction_task_num_per_disk, "2");
-
- // How many rounds of cumulative compaction for each round of base compaction when compaction tasks generation.
- CONF_mInt32(cumulative_compaction_rounds_for_each_base_compaction_round, "9");
-
- // Merge log will be printed for each "row_step_for_compaction_merge_log" rows merged during compaction
- CONF_mInt64(row_step_for_compaction_merge_log, "0");
-
- // Threshold to logging compaction trace, in seconds.
- CONF_mInt32(base_compaction_trace_threshold, "10");
- CONF_mInt32(cumulative_compaction_trace_threshold, "2");
-
- // time interval to record tablet scan count in second for the purpose of calculating tablet scan frequency
- CONF_mInt64(tablet_scan_frequency_time_node_interval_second, "300");
- // coefficient for tablet scan frequency and compaction score when finding a tablet for compaction
- CONF_mInt32(compaction_tablet_scan_frequency_factor, "0");
- CONF_mInt32(compaction_tablet_compaction_score_factor, "1");
-
-
- // Port to start debug webserver on
- CONF_Int32(webserver_port, "8040");
- // Number of webserver workers
- CONF_Int32(webserver_num_workers, "48");
- // Period to update rate counters and sampling counters in ms.
- CONF_mInt32(periodic_counter_update_period_ms, "500");
-
- // Used for mini Load. mini load data file will be removed after this time.
- CONF_Int64(load_data_reserve_hours, "4");
- // log error log will be removed after this time
- CONF_mInt64(load_error_log_reserve_hours, "48");
- CONF_Int32(number_tablet_writer_threads, "16");
-
- // The maximum amount of data that can be processed by a stream load
- CONF_mInt64(streaming_load_max_mb, "10240");
- // Some data formats, such as JSON, cannot be streamed.
- // Therefore, it is necessary to limit the maximum number of
- // such data when using stream load to prevent excessive memory consumption.
- CONF_mInt64(streaming_load_json_max_mb, "100");
- // the alive time of a TabletsChannel.
- // If the channel does not receive any data till this time,
- // the channel will be removed.
- CONF_Int32(streaming_load_rpc_max_alive_time_sec, "1200");
- // the timeout of a rpc to open the tablet writer in remote BE.
- // short operation time, can set a short timeout
- CONF_Int32(tablet_writer_open_rpc_timeout_sec, "60");
- // Deprecated, use query_timeout instead
- // the timeout of a rpc to process one batch in tablet writer.
- // you may need to increase this timeout if using larger 'streaming_load_max_mb',
- // or encounter 'tablet writer write failed' error when loading.
- // CONF_Int32(tablet_writer_rpc_timeout_sec, "600");
- // OlapTableSink sender's send interval, should be less than the real response time of a tablet writer rpc.
- CONF_mInt32(olap_table_sink_send_interval_ms, "10");
-
- // Fragment thread pool
- CONF_Int32(fragment_pool_thread_num_min, "64");
- CONF_Int32(fragment_pool_thread_num_max, "512");
- CONF_Int32(fragment_pool_queue_size, "2048");
-
- //for cast
- // CONF_Bool(cast, "true");
-
- // Spill to disk when query
- // Writable scratch directories, split by ";"
- CONF_String(query_scratch_dirs, "${DORIS_HOME}");
-
- // Control the number of disks on the machine. If 0, this comes from the system settings.
- CONF_Int32(num_disks, "0");
- // The maximum number of the threads per disk is also the max queue depth per disk.
- CONF_Int32(num_threads_per_disk, "0");
- // The read size is the size of the reads sent to os.
- // There is a trade off of latency and throughout, trying to keep disks busy but
- // not introduce seeks. The literature seems to agree that with 8 MB reads, random
- // io and sequential io perform similarly.
- CONF_Int32(read_size, "8388608"); // 8 * 1024 * 1024, Read Size (in bytes)
- CONF_Int32(min_buffer_size, "1024"); // 1024, The minimum read buffer size (in bytes)
-
- // For each io buffer size, the maximum number of buffers the IoMgr will hold onto
- // With 1024B through 8MB buffers, this is up to ~2GB of buffers.
- CONF_Int32(max_free_io_buffers, "128");
-
- CONF_Bool(disable_mem_pools, "false");
-
- // Whether to allocate chunk using mmap. If you enable this, you'd better to
- // increase vm.max_map_count's value whose default value is 65530.
- // you can do it as root via "sysctl -w vm.max_map_count=262144" or
- // "echo 262144 > /proc/sys/vm/max_map_count"
- // NOTE: When this is set to true, you must set chunk_reserved_bytes_limit
- // to a relative large number or the performance is very very bad.
- CONF_Bool(use_mmap_allocate_chunk, "false");
-
- // Chunk Allocator's reserved bytes limit,
- // Default value is 2GB, increase this variable can improve performance, but will
- // acquire more free memory which can not be used by other modules
- CONF_Int64(chunk_reserved_bytes_limit, "2147483648");
-
- // The probing algorithm of partitioned hash table.
- // Enable quadratic probing hash table
- CONF_Bool(enable_quadratic_probing, "false");
-
- // for pprof
- CONF_String(pprof_profile_dir, "${DORIS_HOME}/log");
-
- // for partition
- // CONF_Bool(enable_partitioned_hash_join, "false")
- CONF_Bool(enable_partitioned_aggregation, "true");
-
- // to forward compatibility, will be removed later
- CONF_mBool(enable_token_check, "true");
-
- // to open/close system metrics
- CONF_Bool(enable_system_metrics, "true");
-
- CONF_mBool(enable_prefetch, "true");
-
- // Number of cores Doris will used, this will effect only when it's greater than 0.
- // Otherwise, Doris will use all cores returned from "/proc/cpuinfo".
- CONF_Int32(num_cores, "0");
-
- // CONF_Bool(thread_creation_fault_injection, "false");
-
- // Set this to encrypt and perform an integrity
- // check on all data spilled to disk during a query
- // CONF_Bool(disk_spill_encryption, "false");
-
- // When BE start, If there is a broken disk, BE process will exit by default.
- // Otherwise, we will ignore the broken disk,
- CONF_Bool(ignore_broken_disk, "false");
-
- // Writable scratch directories
- CONF_String(scratch_dirs, "/tmp");
-
- // If false and --scratch_dirs contains multiple directories on the same device,
- // then only the first writable directory is used
- // CONF_Bool(allow_multiple_scratch_dirs_per_device, "false");
-
- // linux transparent huge page
- CONF_Bool(madvise_huge_pages, "false");
-
- // whether use mmap to allocate memory
- CONF_Bool(mmap_buffers, "false");
-
- // max memory can be allocated by buffer pool
- CONF_String(buffer_pool_limit, "80G");
-
- // clean page can be hold by buffer pool
- CONF_String(buffer_pool_clean_pages_limit, "20G");
-
- // Sleep time in seconds between memory maintenance iterations
- CONF_mInt64(memory_maintenance_sleep_time_s, "10");
-
- // Alignment
- CONF_Int32(memory_max_alignment, "16");
+// Dir of custom config file
+CONF_String(custom_config_dir, "${DORIS_HOME}/conf");
+
+// cluster id
+CONF_Int32(cluster_id, "-1");
+// port on which BackendService is exported
+CONF_Int32(be_port, "9060");
+
+// port for brpc
+CONF_Int32(brpc_port, "8060");
+
+// the number of bthreads for brpc, the default value is set to -1, which means the number of bthreads is #cpu-cores
+CONF_Int32(brpc_num_threads, "-1")
+
+ // Declare a selection strategy for those servers have many ips.
+ // Note that there should at most one ip match this list.
+ // this is a list in semicolon-delimited format, in CIDR notation, e.g. 10.10.10.0/24
+ // If no ip match this rule, will choose one randomly.
+ CONF_String(priority_networks, "");
+
+////
+//// tcmalloc gc parameter
+////
+// min memory for TCmalloc, when used memory is smaller than this, do not returned to OS
+CONF_mInt64(tc_use_memory_min, "10737418240");
+// free memory rate.[0-100]
+CONF_mInt64(tc_free_memory_rate, "20");
+
+// Bound on the total amount of bytes allocated to thread caches.
+// This bound is not strict, so it is possible for the cache to go over this bound
+// in certain circumstances. This value defaults to 1GB
+// If you suspect your application is not scaling to many threads due to lock contention in TCMalloc,
+// you can try increasing this value. This may improve performance, at a cost of extra memory
+// use by TCMalloc.
+// reference: https://gperftools.github.io/gperftools/tcmalloc.html: TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES
+// https://github.com/gperftools/gperftools/issues/1111
+CONF_Int64(tc_max_total_thread_cache_bytes, "1073741824");
+
+// process memory limit specified as number of bytes
+// ('<int>[bB]?'), megabytes ('<float>[mM]'), gigabytes ('<float>[gG]'),
+// or percentage of the physical memory ('<int>%').
+// defaults to bytes if no unit is given"
+// must larger than 0. and if larger than physical memory size,
+// it will be set to physical memory size.
+CONF_String(mem_limit, "80%");
+
+// the port heartbeat service used
+CONF_Int32(heartbeat_service_port, "9050");
+// the count of heart beat service
+CONF_Int32(heartbeat_service_thread_count, "1");
+// the count of thread to create table
+CONF_Int32(create_tablet_worker_count, "3");
+// the count of thread to drop table
+CONF_Int32(drop_tablet_worker_count, "3");
+// the count of thread to batch load
+CONF_Int32(push_worker_count_normal_priority, "3");
+// the count of thread to high priority batch load
+CONF_Int32(push_worker_count_high_priority, "3");
+// the count of thread to publish version
+CONF_Int32(publish_version_worker_count, "8");
+// the count of thread to clear transaction task
+CONF_Int32(clear_transaction_task_worker_count, "1");
+// the count of thread to delete
+CONF_Int32(delete_worker_count, "3");
+// the count of thread to alter table
+CONF_Int32(alter_tablet_worker_count, "3");
+// the count of thread to clone
+CONF_Int32(clone_worker_count, "3");
+// the count of thread to clone
+CONF_Int32(storage_medium_migrate_count, "1");
+// the count of thread to check consistency
+CONF_Int32(check_consistency_worker_count, "1");
+// the count of thread to upload
+CONF_Int32(upload_worker_count, "1");
+// the count of thread to download
+CONF_Int32(download_worker_count, "1");
+// the count of thread to make snapshot
+CONF_Int32(make_snapshot_worker_count, "5");
+// the count of thread to release snapshot
+CONF_Int32(release_snapshot_worker_count, "5");
+// the interval time(seconds) for agent report tasks signatrue to FE
+CONF_mInt32(report_task_interval_seconds, "10");
+// the interval time(seconds) for agent report disk state to FE
+CONF_mInt32(report_disk_state_interval_seconds, "60");
+// the interval time(seconds) for agent report olap table to FE
+CONF_mInt32(report_tablet_interval_seconds, "60");
+// the interval time(seconds) for agent report plugin status to FE
+// CONF_Int32(report_plugin_interval_seconds, "120");
+// the timeout(seconds) for alter table
+// CONF_Int32(alter_tablet_timeout_seconds, "86400");
+// the timeout(seconds) for make snapshot
+// CONF_Int32(make_snapshot_timeout_seconds, "600");
+// the timeout(seconds) for release snapshot
+// CONF_Int32(release_snapshot_timeout_seconds, "600");
+// the max download speed(KB/s)
+CONF_mInt32(max_download_speed_kbps, "50000");
+// download low speed limit(KB/s)
+CONF_mInt32(download_low_speed_limit_kbps, "50");
+// download low speed time(seconds)
+CONF_mInt32(download_low_speed_time, "300");
+// curl verbose mode
+// CONF_Int64(curl_verbose_mode, "1");
+// seconds to sleep for each time check table status
+// CONF_Int32(check_status_sleep_time_seconds, "10");
+// sleep time for one second
+CONF_Int32(sleep_one_second, "1");
+// sleep time for five seconds
+CONF_Int32(sleep_five_seconds, "5");
+
+// log dir
+CONF_String(sys_log_dir, "${DORIS_HOME}/log");
+CONF_String(user_function_dir, "${DORIS_HOME}/lib/udf");
+// INFO, WARNING, ERROR, FATAL
+CONF_String(sys_log_level, "INFO");
+// TIME-DAY, TIME-HOUR, SIZE-MB-nnn
+CONF_String(sys_log_roll_mode, "SIZE-MB-1024");
+// log roll num
+CONF_Int32(sys_log_roll_num, "10");
+// verbose log
+CONF_Strings(sys_log_verbose_modules, "");
+// verbose log level
+CONF_Int32(sys_log_verbose_level, "10");
+// log buffer level
+CONF_String(log_buffer_level, "");
+
+// Pull load task dir
+CONF_String(pull_load_task_dir, "${DORIS_HOME}/var/pull_load");
+
+// the maximum number of bytes to display on the debug webserver's log page
+CONF_Int64(web_log_bytes, "1048576");
+// number of threads available to serve backend execution requests
+CONF_Int32(be_service_threads, "64");
+// key=value pair of default query options for Doris, separated by ','
+CONF_String(default_query_options, "");
+
+// If non-zero, Doris will output memory usage every log_mem_usage_interval'th fragment completion.
+// CONF_Int32(log_mem_usage_interval, "0");
+
+// cgroups allocated for doris
+CONF_String(doris_cgroups, "");
+
+// Controls the number of threads to run work per core. It's common to pick 2x
+// or 3x the number of cores. This keeps the cores busy without causing excessive
+// thrashing.
+CONF_Int32(num_threads_per_core, "3");
+// if true, compresses tuple data in Serialize
+CONF_Bool(compress_rowbatches, "true");
+// serialize and deserialize each returned row batch
+CONF_Bool(serialize_batch, "false");
+// interval between profile reports; in seconds
+CONF_mInt32(status_report_interval, "5");
+// Local directory to copy UDF libraries from HDFS into
+CONF_String(local_library_dir, "${UDF_RUNTIME_DIR}");
+// number of olap scanner thread pool size
+CONF_Int32(doris_scanner_thread_pool_thread_num, "48");
+// number of olap scanner thread pool queue size
+CONF_Int32(doris_scanner_thread_pool_queue_size, "102400");
+// number of etl thread pool size
+CONF_Int32(etl_thread_pool_size, "8");
+// number of etl thread pool size
+CONF_Int32(etl_thread_pool_queue_size, "256");
+// port on which to run Doris test backend
+CONF_Int32(port, "20001");
+// default thrift client connect timeout(in seconds)
+CONF_Int32(thrift_connect_timeout_seconds, "3");
+// default thrift client retry interval (in milliseconds)
+CONF_mInt64(thrift_client_retry_interval_ms, "1000");
+// max row count number for single scan range
+CONF_mInt32(doris_scan_range_row_count, "524288");
+// size of scanner queue between scanner thread and compute thread
+CONF_mInt32(doris_scanner_queue_size, "1024");
+// single read execute fragment row size
+CONF_mInt32(doris_scanner_row_num, "16384");
+// number of max scan keys
+CONF_mInt32(doris_max_scan_key_num, "1024");
+// the max number of push down values of a single column.
+// if exceed, no conditions will be pushed down for that column.
+CONF_mInt32(max_pushdown_conditions_per_column, "1024");
+// return_row / total_row
+CONF_mInt32(doris_max_pushdown_conjuncts_return_rate, "90");
+// (Advanced) Maximum size of per-query receive-side buffer
+CONF_mInt32(exchg_node_buffer_size_bytes, "10485760");
+// insert sort threshold for sorter
+// CONF_Int32(insertion_threshold, "16");
+// the block_size every block allocate for sorter
+CONF_Int32(sorter_block_size, "8388608");
+// push_write_mbytes_per_sec
+CONF_Int32(push_write_mbytes_per_sec, "10");
+
+CONF_mInt64(column_dictionary_key_ratio_threshold, "0");
+CONF_mInt64(column_dictionary_key_size_threshold, "0");
+// if true, output IR after optimization passes
+// CONF_Bool(dump_ir, "false");
+// if set, saves the generated IR to the output file.
+//CONF_String(module_output, "");
+// memory_limitation_per_thread_for_schema_change unit GB
+CONF_mInt32(memory_limitation_per_thread_for_schema_change, "2");
+
+// CONF_Int64(max_unpacked_row_block_size, "104857600");
+
+CONF_mInt32(file_descriptor_cache_clean_interval, "3600");
+CONF_mInt32(disk_stat_monitor_interval, "5");
+CONF_mInt32(unused_rowset_monitor_interval, "30");
+CONF_String(storage_root_path, "${DORIS_HOME}/storage");
+
+// Config is used to check incompatible old format hdr_ format
+// whether doris uses strict way. When config is true, process will log fatal
+// and exit. When config is false, process will only log warning.
+CONF_Bool(storage_strict_check_incompatible_old_format, "true");
+
+// BE process will exit if the percentage of error disk reach this value.
+CONF_mInt32(max_percentage_of_error_disk, "0");
+// CONF_Int32(default_num_rows_per_data_block, "1024");
+CONF_mInt32(default_num_rows_per_column_file_block, "1024");
+// pending data policy
+CONF_mInt32(pending_data_expire_time_sec, "1800");
+// inc_rowset expired interval
+CONF_mInt32(inc_rowset_expired_sec, "1800");
+// inc_rowset snapshot rs sweep time interval
+CONF_mInt32(tablet_rowset_stale_sweep_time_sec, "1800");
+// garbage sweep policy
+CONF_Int32(max_garbage_sweep_interval, "3600");
+CONF_Int32(min_garbage_sweep_interval, "180");
+CONF_mInt32(snapshot_expire_time_sec, "172800");
+// 仅仅是建议值，当磁盘空间不足时，trash下的文件保存期可不遵守这个参数
+CONF_mInt32(trash_file_expire_time_sec, "259200");
+// check row nums for BE/CE and schema change. true is open, false is closed.
+CONF_mBool(row_nums_check, "true");
+//file descriptors cache, by default, cache 32768 descriptors
+CONF_Int32(file_descriptor_cache_capacity, "32768");
+// minimum file descriptor number
+// modify them upon necessity
+CONF_Int32(min_file_descriptor_number, "60000");
+CONF_Int64(index_stream_cache_capacity, "10737418240");
+// CONF_Int64(max_packed_row_block_size, "20971520");
+
+// Cache for storage page size
+CONF_String(storage_page_cache_limit, "20G");
+// whether to disable page cache feature in storage
+CONF_Bool(disable_storage_page_cache, "false");
+
+// be policy
+// whether disable automatic compaction task
+CONF_mBool(disable_auto_compaction, "false");
+// check the configuration of auto compaction in seconds when auto compaction disabled
+CONF_mInt32(check_auto_compaction_interval_seconds, "5");
+
+// CONF_Int64(base_compaction_start_hour, "20");
+// CONF_Int64(base_compaction_end_hour, "7");
+CONF_mInt64(base_compaction_num_cumulative_deltas, "5");
+CONF_mDouble(base_cumulative_delta_ratio, "0.3");
+CONF_mInt64(base_compaction_interval_seconds_since_last_operation, "86400");
+CONF_mInt32(base_compaction_write_mbytes_per_sec, "5");
+
+// config the cumulative compaction policy
+// Valid configs: num_base, size_based
+// num_based policy, the original version of cumulative compaction, cumulative version compaction once.
+// size_based policy, a optimization version of cumulative compaction, targeting the use cases requiring
+// lower write amplification, trading off read amplification and space amplification.
+CONF_String(cumulative_compaction_policy, "size_based");
+
+// In size_based policy, output rowset of cumulative compaction total disk size exceed this config size,
+// this rowset will be given to base compaction, unit is m byte.
+CONF_mInt64(cumulative_size_based_promotion_size_mbytes, "1024");
+// In size_based policy, output rowset of cumulative compaction total disk size exceed this config ratio of
+// base rowset's total disk size, this rowset will be given to base compaction. The value must be between
+// 0 and 1.
+CONF_mDouble(cumulative_size_based_promotion_ratio, "0.05");
+// In size_based policy, the smallest size of rowset promotion. When the rowset is less than this config, this
+// rowset will be not given to base compaction. The unit is m byte.
+CONF_mInt64(cumulative_size_based_promotion_min_size_mbytes, "64");
+// The lower bound size to do cumulative compaction. When total disk size of candidate rowsets is less than
+// this size, size_based policy may not do to cumulative compaction. The unit is m byte.
+CONF_mInt64(cumulative_size_based_compaction_lower_size_mbytes, "64");
+
+// cumulative compaction policy: min and max delta file's number
+CONF_mInt64(min_cumulative_compaction_num_singleton_deltas, "5");
+CONF_mInt64(max_cumulative_compaction_num_singleton_deltas, "1000");
+CONF_mInt64(cumulative_compaction_budgeted_bytes, "104857600");
+// CONF_Int32(cumulative_compaction_write_mbytes_per_sec, "100");
+// cumulative compaction skips recently published deltas in order to prevent
+// compacting a version that might be queried (in case the query planning phase took some time).
+// the following config set the window size
+CONF_mInt32(cumulative_compaction_skip_window_seconds, "30");
+
+// if compaction of a tablet failed, this tablet should not be chosen to
+// compaction until this interval passes.
+CONF_mInt64(min_compaction_failure_interval_sec, "600"); // 10 min
+
+// This config can be set to limit thread number in compaction thread pool.
+CONF_mInt32(min_compaction_threads, "10");
+CONF_mInt32(max_compaction_threads, "10");
+
+// The upper limit of "permits" held by all compaction tasks. This config can be set to limit memory consumption for compaction.
+CONF_mInt64(total_permits_for_compaction_score, "10000");
+
+// Compaction task number per disk.
+CONF_mInt32(compaction_task_num_per_disk, "2");
+
+// How many rounds of cumulative compaction for each round of base compaction when compaction tasks generation.
+CONF_mInt32(cumulative_compaction_rounds_for_each_base_compaction_round, "9");
+
+// Merge log will be printed for each "row_step_for_compaction_merge_log" rows merged during compaction
+CONF_mInt64(row_step_for_compaction_merge_log, "0");
+
+// Threshold to logging compaction trace, in seconds.
+CONF_mInt32(base_compaction_trace_threshold, "10");
+CONF_mInt32(cumulative_compaction_trace_threshold, "2");
+
+// time interval to record tablet scan count in second for the purpose of calculating tablet scan frequency
+CONF_mInt64(tablet_scan_frequency_time_node_interval_second, "300");
+// coefficient for tablet scan frequency and compaction score when finding a tablet for compaction
+CONF_mInt32(compaction_tablet_scan_frequency_factor, "0");
+CONF_mInt32(compaction_tablet_compaction_score_factor, "1");
+
+// Port to start debug webserver on
+CONF_Int32(webserver_port, "8040");
+// Number of webserver workers
+CONF_Int32(webserver_num_workers, "48");
+// Period to update rate counters and sampling counters in ms.
+CONF_mInt32(periodic_counter_update_period_ms, "500");
+
+// Used for mini Load. mini load data file will be removed after this time.
+CONF_Int64(load_data_reserve_hours, "4");
+// log error log will be removed after this time
+CONF_mInt64(load_error_log_reserve_hours, "48");
+CONF_Int32(number_tablet_writer_threads, "16");
+
+// The maximum amount of data that can be processed by a stream load
+CONF_mInt64(streaming_load_max_mb, "10240");
+// Some data formats, such as JSON, cannot be streamed.
+// Therefore, it is necessary to limit the maximum number of
+// such data when using stream load to prevent excessive memory consumption.
+CONF_mInt64(streaming_load_json_max_mb, "100");
+// the alive time of a TabletsChannel.
+// If the channel does not receive any data till this time,
+// the channel will be removed.
+CONF_Int32(streaming_load_rpc_max_alive_time_sec, "1200");
+// the timeout of a rpc to open the tablet writer in remote BE.
+// short operation time, can set a short timeout
+CONF_Int32(tablet_writer_open_rpc_timeout_sec, "60");
+// Deprecated, use query_timeout instead
+// the timeout of a rpc to process one batch in tablet writer.
+// you may need to increase this timeout if using larger 'streaming_load_max_mb',
+// or encounter 'tablet writer write failed' error when loading.
+// CONF_Int32(tablet_writer_rpc_timeout_sec, "600");
+// OlapTableSink sender's send interval, should be less than the real response time of a tablet writer rpc.
+CONF_mInt32(olap_table_sink_send_interval_ms, "10");
+
+// Fragment thread pool
+CONF_Int32(fragment_pool_thread_num_min, "64");
+CONF_Int32(fragment_pool_thread_num_max, "512");
+CONF_Int32(fragment_pool_queue_size, "2048");
+
+//for cast
+// CONF_Bool(cast, "true");
+
+// Spill to disk when query
+// Writable scratch directories, split by ";"
+CONF_String(query_scratch_dirs, "${DORIS_HOME}");
+
+// Control the number of disks on the machine. If 0, this comes from the system settings.
+CONF_Int32(num_disks, "0");
+// The maximum number of the threads per disk is also the max queue depth per disk.
+CONF_Int32(num_threads_per_disk, "0");
+// The read size is the size of the reads sent to os.
+// There is a trade off of latency and throughout, trying to keep disks busy but
+// not introduce seeks. The literature seems to agree that with 8 MB reads, random
+// io and sequential io perform similarly.
+CONF_Int32(read_size, "8388608"); // 8 * 1024 * 1024, Read Size (in bytes)
+CONF_Int32(min_buffer_size, "1024"); // 1024, The minimum read buffer size (in bytes)
+
+// For each io buffer size, the maximum number of buffers the IoMgr will hold onto
+// With 1024B through 8MB buffers, this is up to ~2GB of buffers.
+CONF_Int32(max_free_io_buffers, "128");
+
+CONF_Bool(disable_mem_pools, "false");
+
+// Whether to allocate chunk using mmap. If you enable this, you'd better to
+// increase vm.max_map_count's value whose default value is 65530.
+// you can do it as root via "sysctl -w vm.max_map_count=262144" or
+// "echo 262144 > /proc/sys/vm/max_map_count"
+// NOTE: When this is set to true, you must set chunk_reserved_bytes_limit
+// to a relative large number or the performance is very very bad.
+CONF_Bool(use_mmap_allocate_chunk, "false");
+
+// Chunk Allocator's reserved bytes limit,
+// Default value is 2GB, increase this variable can improve performance, but will
+// acquire more free memory which can not be used by other modules
+CONF_Int64(chunk_reserved_bytes_limit, "2147483648");
+
+// The probing algorithm of partitioned hash table.
+// Enable quadratic probing hash table
+CONF_Bool(enable_quadratic_probing, "false");
+
+// for pprof
+CONF_String(pprof_profile_dir, "${DORIS_HOME}/log");
+
+// for partition
+// CONF_Bool(enable_partitioned_hash_join, "false")
+CONF_Bool(enable_partitioned_aggregation, "true");
+
+// to forward compatibility, will be removed later
+CONF_mBool(enable_token_check, "true");
+
+// to open/close system metrics
+CONF_Bool(enable_system_metrics, "true");
+
+CONF_mBool(enable_prefetch, "true");
+
+// Number of cores Doris will used, this will effect only when it's greater than 0.
+// Otherwise, Doris will use all cores returned from "/proc/cpuinfo".
+CONF_Int32(num_cores, "0");
+
+// CONF_Bool(thread_creation_fault_injection, "false");
+
+// Set this to encrypt and perform an integrity
+// check on all data spilled to disk during a query
+// CONF_Bool(disk_spill_encryption, "false");
+
+// When BE start, If there is a broken disk, BE process will exit by default.
+// Otherwise, we will ignore the broken disk,
+CONF_Bool(ignore_broken_disk, "false");
+
+// Writable scratch directories
+CONF_String(scratch_dirs, "/tmp");
+
+// If false and --scratch_dirs contains multiple directories on the same device,
+// then only the first writable directory is used
+// CONF_Bool(allow_multiple_scratch_dirs_per_device, "false");
+
+// linux transparent huge page
+CONF_Bool(madvise_huge_pages, "false");
+
+// whether use mmap to allocate memory
+CONF_Bool(mmap_buffers, "false");
+
+// max memory can be allocated by buffer pool
+CONF_String(buffer_pool_limit, "80G");
+
+// clean page can be hold by buffer pool
+CONF_String(buffer_pool_clean_pages_limit, "20G");
+
+// Sleep time in seconds between memory maintenance iterations
+CONF_mInt64(memory_maintenance_sleep_time_s, "10");
+
+// Alignment
+CONF_Int32(memory_max_alignment, "16");
- // write buffer size before flush
- CONF_mInt64(write_buffer_size, "104857600");
+// write buffer size before flush
+CONF_mInt64(write_buffer_size, "104857600");
- // following 2 configs limit the memory consumption of load process on a Backend.
- // eg: memory limit to 80% of mem limit config but up to 100GB(default)
- // NOTICE(cmy): set these default values very large because we don't want to
- // impact the load performance when user upgrading Doris.
- // user should set these configs properly if necessary.
- CONF_Int64(load_process_max_memory_limit_bytes, "107374182400"); // 100GB
- CONF_Int32(load_process_max_memory_limit_percent, "80"); // 80%
+// following 2 configs limit the memory consumption of load process on a Backend.
+// eg: memory limit to 80% of mem limit config but up to 100GB(default)
+// NOTICE(cmy): set these default values very large because we don't want to
+// impact the load performance when user upgrading Doris.
+// user should set these configs properly if necessary.
+CONF_Int64(load_process_max_memory_limit_bytes, "107374182400"); // 100GB
+CONF_Int32(load_process_max_memory_limit_percent, "80"); // 80%
- // update interval of tablet stat cache
- CONF_mInt32(tablet_stat_cache_update_interval_second, "300");
+// update interval of tablet stat cache
+CONF_mInt32(tablet_stat_cache_update_interval_second, "300");
- // result buffer cancelled time (unit: second)
- CONF_mInt32(result_buffer_cancelled_interval_time, "300");
+// result buffer cancelled time (unit: second)
+CONF_mInt32(result_buffer_cancelled_interval_time, "300");
- // the increased frequency of priority for remaining tasks in BlockingPriorityQueue
- CONF_mInt32(priority_queue_remaining_tasks_increased_frequency, "512");
+// the increased frequency of priority for remaining tasks in BlockingPriorityQueue
+CONF_mInt32(priority_queue_remaining_tasks_increased_frequency, "512");
- // sync tablet_meta when modifying meta
- CONF_mBool(sync_tablet_meta, "false");
+// sync tablet_meta when modifying meta
+CONF_mBool(sync_tablet_meta, "false");
- // default thrift rpc timeout ms
- CONF_mInt32(thrift_rpc_timeout_ms, "5000");
+// default thrift rpc timeout ms
+CONF_mInt32(thrift_rpc_timeout_ms, "5000");
- // txn commit rpc timeout
- CONF_mInt32(txn_commit_rpc_timeout_ms, "10000");
+// txn commit rpc timeout
+CONF_mInt32(txn_commit_rpc_timeout_ms, "10000");
- // If set to true, metric calculator will run
- CONF_Bool(enable_metric_calculator, "true");
+// If set to true, metric calculator will run
+CONF_Bool(enable_metric_calculator, "true");
- // max consumer num in one data consumer group, for routine load
- CONF_mInt32(max_consumer_num_per_group, "3");
+// max consumer num in one data consumer group, for routine load
+CONF_mInt32(max_consumer_num_per_group, "3");
- // the size of thread pool for routine load task.
- // this should be larger than FE config 'max_concurrent_task_num_per_be' (default 5)
- CONF_Int32(routine_load_thread_pool_size, "10");
+// the size of thread pool for routine load task.
+// this should be larger than FE config 'max_concurrent_task_num_per_be' (default 5)
+CONF_Int32(routine_load_thread_pool_size, "10");
- // Is set to true, index loading failure will not causing BE exit,
- // and the tablet will be marked as bad, so that FE will try to repair it.
- // CONF_Bool(auto_recover_index_loading_failure, "false");
+// Is set to true, index loading failure will not causing BE exit,
+// and the tablet will be marked as bad, so that FE will try to repair it.
+// CONF_Bool(auto_recover_index_loading_failure, "false");
- // max external scan cache batch count, means cache max_memory_cache_batch_count * batch_size row
- // default is 20, batch_size's default value is 1024 means 20 * 1024 rows will be cached
- CONF_mInt32(max_memory_sink_batch_count, "20");
+// max external scan cache batch count, means cache max_memory_cache_batch_count * batch_size row
+// default is 20, batch_size's default value is 1024 means 20 * 1024 rows will be cached
+CONF_mInt32(max_memory_sink_batch_count, "20");
- // This configuration is used for the context gc thread schedule period
- // note: unit is minute, default is 5min
- CONF_mInt32(scan_context_gc_interval_min, "5");
+// This configuration is used for the context gc thread schedule period
+// note: unit is minute, default is 5min
+CONF_mInt32(scan_context_gc_interval_min, "5");
- // es scroll keep-alive
- CONF_String(es_scroll_keepalive, "5m");
+// es scroll keep-alive
+CONF_String(es_scroll_keepalive, "5m");
- // HTTP connection timeout for es
- CONF_Int32(es_http_timeout_ms, "5000");
+// HTTP connection timeout for es
+CONF_Int32(es_http_timeout_ms, "5000");
- // the max client cache number per each host
- // There are variety of client cache in BE, but currently we use the
- // same cache size configuration.
- // TODO(cmy): use different config to set different client cache if necessary.
- CONF_Int32(max_client_cache_size_per_host, "10");
+// the max client cache number per each host
+// There are variety of client cache in BE, but currently we use the
+// same cache size configuration.
+// TODO(cmy): use different config to set different client cache if necessary.
+CONF_Int32(max_client_cache_size_per_host, "10");
- // Dir to save files downloaded by SmallFileMgr
- CONF_String(small_file_dir, "${DORIS_HOME}/lib/small_file/");
- // path gc
- CONF_Bool(path_gc_check, "true");
- CONF_Int32(path_gc_check_interval_second, "86400");
- CONF_mInt32(path_gc_check_step, "1000");
- CONF_mInt32(path_gc_check_step_interval_ms, "10");
- CONF_mInt32(path_scan_interval_second, "86400");
+// Dir to save files downloaded by SmallFileMgr
+CONF_String(small_file_dir, "${DORIS_HOME}/lib/small_file/");
+// path gc
+CONF_Bool(path_gc_check, "true");
+CONF_Int32(path_gc_check_interval_second, "86400");
+CONF_mInt32(path_gc_check_step, "1000");
+CONF_mInt32(path_gc_check_step_interval_ms, "10");
+CONF_mInt32(path_scan_interval_second, "86400");
- // The following 2 configs limit the max usage of disk capacity of a data dir.
- // If both of these 2 threshold reached, no more data can be writen into that data dir.
- // The percent of max used capacity of a data dir
- CONF_mInt32(storage_flood_stage_usage_percent, "95"); // 95%
- // The min bytes that should be left of a data dir
- CONF_mInt64(storage_flood_stage_left_capacity_bytes, "1073741824"); // 1GB
- // number of thread for flushing memtable per store
- CONF_Int32(flush_thread_num_per_store, "2");
+// The following 2 configs limit the max usage of disk capacity of a data dir.
+// If both of these 2 threshold reached, no more data can be writen into that data dir.
+// The percent of max used capacity of a data dir
+CONF_mInt32(storage_flood_stage_usage_percent, "95"); // 95%
+// The min bytes that should be left of a data dir
+CONF_mInt64(storage_flood_stage_left_capacity_bytes, "1073741824"); // 1GB
+// number of thread for flushing memtable per store
+CONF_Int32(flush_thread_num_per_store, "2");
- // config for tablet meta checkpoint
- CONF_mInt32(tablet_meta_checkpoint_min_new_rowsets_num, "10");
- CONF_mInt32(tablet_meta_checkpoint_min_interval_secs, "600");
+// config for tablet meta checkpoint
+CONF_mInt32(tablet_meta_checkpoint_min_new_rowsets_num, "10");
+CONF_mInt32(tablet_meta_checkpoint_min_interval_secs, "600");
- // config for default rowset type
- // Valid configs: ALPHA, BETA
- CONF_String(default_rowset_type, "BETA");
+// config for default rowset type
+// Valid configs: ALPHA, BETA
+CONF_String(default_rowset_type, "BETA");
- // Maximum size of a single message body in all protocols
- CONF_Int64(brpc_max_body_size, "209715200");
- // Max unwritten bytes in each socket, if the limit is reached, Socket.Write fails with EOVERCROWDED
- CONF_Int64(brpc_socket_max_unwritten_bytes, "67108864");
+// Maximum size of a single message body in all protocols
+CONF_Int64(brpc_max_body_size, "209715200");
+// Max unwritten bytes in each socket, if the limit is reached, Socket.Write fails with EOVERCROWDED
+CONF_Int64(brpc_socket_max_unwritten_bytes, "67108864");
+
+// max number of txns for every txn_partition_map in txn manager
+// this is a self protection to avoid too many txns saving in manager
+CONF_mInt64(max_runnings_transactions_per_txn_map, "100");
+
+// tablet_map_lock shard size, the value is 2^n, n=0,1,2,3,4
+// this is a an enhancement for better performance to manage tablet
+CONF_Int32(tablet_map_shard_size, "1");
+
+CONF_String(plugin_path, "${DORIS_HOME}/plugin");
+
+// txn_map_lock shard size, the value is 2^n, n=0,1,2,3,4
+// this is a an enhancement for better performance to manage txn
+CONF_Int32(txn_map_shard_size, "128");
+
+// txn_lock shard size, the value is 2^n, n=0,1,2,3,4
+// this is a an enhancement for better performance to commit and publish txn
+CONF_Int32(txn_shard_size, "1024");
+
+// Whether to continue to start be when load tablet from header failed.
+CONF_Bool(ignore_load_tablet_failure, "false");
+
+// Whether to continue to start be when load tablet from header failed.
+CONF_Bool(ignore_rowset_stale_unconsistent_delete, "false");
+
+// Soft memory limit as a fraction of hard memory limit.
+CONF_Double(soft_mem_limit_frac, "0.9");
+
+// Set max cache's size of query results, the unit is M byte
+CONF_Int32(query_cache_max_size_mb, "256");
+
+// Cache memory is pruned when reach query_cache_max_size_mb + query_cache_elasticity_size_mb
+CONF_Int32(query_cache_elasticity_size_mb, "128");
+
+// Maximum number of cache partitions corresponding to a SQL
+CONF_Int32(query_cache_max_partition_count, "1024");
+
+// Maximum number of version of a tablet. If the version num of a tablet exceed limit,
+// the load process will reject new incoming load job of this tablet.
+// This is to avoid too many version num.
+CONF_mInt32(max_tablet_version_num, "500");
+
+// Frontend mainly use two thrift sever type: THREAD_POOL, THREADED. if fe use THREADED model for thrift server,
+// the thrift_server_type_of_fe should be set THREADED to make be thrift client to fe constructed with TFramedTransport
+CONF_String(thrift_server_type_of_fe, "THREAD_POOL");
- // max number of txns for every txn_partition_map in txn manager
- // this is a self protection to avoid too many txns saving in manager
- CONF_mInt64(max_runnings_transactions_per_txn_map, "100");
-
- // tablet_map_lock shard size, the value is 2^n, n=0,1,2,3,4
- // this is a an enhancement for better performance to manage tablet
- CONF_Int32(tablet_map_shard_size, "1");
-
- CONF_String(plugin_path, "${DORIS_HOME}/plugin");
-
- // txn_map_lock shard size, the value is 2^n, n=0,1,2,3,4
- // this is a an enhancement for better performance to manage txn
- CONF_Int32(txn_map_shard_size, "128");
-
- // txn_lock shard size, the value is 2^n, n=0,1,2,3,4
- // this is a an enhancement for better performance to commit and publish txn
- CONF_Int32(txn_shard_size, "1024");
-
- // Whether to continue to start be when load tablet from header failed.
- CONF_Bool(ignore_load_tablet_failure, "false");
-
- // Whether to continue to start be when load tablet from header failed.
- CONF_Bool(ignore_rowset_stale_unconsistent_delete, "false");
-
- // Soft memory limit as a fraction of hard memory limit.
- CONF_Double(soft_mem_limit_frac, "0.9");
-
- // Set max cache's size of query results, the unit is M byte
- CONF_Int32(query_cache_max_size_mb, "256");
-
- // Cache memory is pruned when reach query_cache_max_size_mb + query_cache_elasticity_size_mb
- CONF_Int32(query_cache_elasticity_size_mb, "128");
-
- // Maximum number of cache partitions corresponding to a SQL
- CONF_Int32(query_cache_max_partition_count, "1024");
-
- // Maximum number of version of a tablet. If the version num of a tablet exceed limit,
- // the load process will reject new incoming load job of this tablet.
- // This is to avoid too many version num.
- CONF_mInt32(max_tablet_version_num, "500");
-
- // Frontend mainly use two thrift sever type: THREAD_POOL, THREADED. if fe use THREADED model for thrift server,
- // the thrift_server_type_of_fe should be set THREADED to make be thrift client to fe constructed with TFramedTransport
- CONF_String(thrift_server_type_of_fe, "THREAD_POOL");
-
} // namespace config
} // namespace doris
diff --git a/be/src/common/configbase.cpp b/be/src/common/configbase.cpp
index c46c34b..1a8f97a 100644
--- a/be/src/common/configbase.cpp
+++ b/be/src/common/configbase.cpp
@@ -246,8 +246,8 @@ void Properties::set(const std::string& key, const std::string& val) {
}
bool Properties::dump(const std::string& conffile) {
- std::vector<std::string> files = { conffile };
- Status st = FileSystemUtil::remove_paths(files);
+ std::vector<std::string> files = {conffile};
+ Status st = FileSystemUtil::remove_paths(files);
if (!st.ok()) {
return false;
}
@@ -261,7 +261,7 @@ bool Properties::dump(const std::string& conffile) {
out << "# You can modify this file manually, and the configurations in this file\n";
out << "# will overwrite the configurations in be.conf\n";
out << "\n";
-
+
for (auto const& iter : file_conf_map) {
out << iter.first << " = " << iter.second << "\n";
}
@@ -291,20 +291,19 @@ std::ostream& operator<<(std::ostream& out, const std::vector<T>& v) {
return out;
}
-#define SET_FIELD(FIELD, TYPE, FILL_CONFMAP, SET_TO_DEFAULT) \
- if (strcmp((FIELD).type, #TYPE) == 0) { \
- if (!props.get_or_default( \
- (FIELD).name, ((SET_TO_DEFAULT) ? (FIELD).defval : nullptr), \
- *reinterpret_cast<TYPE*>((FIELD).storage))) { \
- std::cerr << "config field error: " << (FIELD).name << std::endl; \
- return false; \
- } \
- if (FILL_CONFMAP) { \
- std::ostringstream oss; \
- oss << (*reinterpret_cast<TYPE*>((FIELD).storage)); \
- (*full_conf_map)[(FIELD).name] = oss.str(); \
- } \
- continue; \
+#define SET_FIELD(FIELD, TYPE, FILL_CONFMAP, SET_TO_DEFAULT) \
+ if (strcmp((FIELD).type, #TYPE) == 0) { \
+ if (!props.get_or_default((FIELD).name, ((SET_TO_DEFAULT) ? (FIELD).defval : nullptr), \
+ *reinterpret_cast<TYPE*>((FIELD).storage))) { \
+ std::cerr << "config field error: " << (FIELD).name << std::endl; \
+ return false; \
+ } \
+ if (FILL_CONFMAP) { \
+ std::ostringstream oss; \
+ oss << (*reinterpret_cast<TYPE*>((FIELD).storage)); \
+ (*full_conf_map)[(FIELD).name] = oss.str(); \
+ } \
+ continue; \
}
// init conf fields
@@ -355,12 +354,11 @@ bool init(const char* conf_file, bool fillconfmap, bool must_exist, bool set_to_
return Status::OK(); \
}
-
// write config to be_custom.conf
// the caller need to make sure that the given config is valid
bool persist_config(const std::string& field, const std::string& value) {
// lock to make sure only one thread can modify the be_custom.conf
- std::lock_guard<std::mutex> l(custom_conf_lock);
+ std::lock_guard<std::mutex> l(custom_conf_lock);
static const string conffile = string(getenv("DORIS_HOME")) + "/conf/be_custom.conf";
Status st = FileSystemUtil::create_file(conffile);
diff --git a/be/src/common/configbase.h b/be/src/common/configbase.h
index 93c06ef..80b690c 100644
--- a/be/src/common/configbase.h
+++ b/be/src/common/configbase.h
@@ -137,7 +137,8 @@ extern std::mutex custom_conf_lock;
// If fillconfmap is true, the updated config will also update the `full_conf_map`.
// If must_exist is true and `conf_file` does not exist, this function will return false.
// If set_to_default is true, the config value will be set to default value if not found in `conf_file`.
-bool init(const char* conf_file, bool fillconfmap = false, bool must_exist = true, bool set_to_default = true);
+bool init(const char* conf_file, bool fillconfmap = false, bool must_exist = true,
+ bool set_to_default = true);
Status set_config(const std::string& field, const std::string& value, bool need_persist = false);
diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp
index 5b045c0..c05ef99 100644
--- a/be/src/common/daemon.cpp
+++ b/be/src/common/daemon.cpp
@@ -21,44 +21,44 @@
#include <gperftools/malloc_extension.h>
#include "common/config.h"
-#include "util/cpu_info.h"
-#include "util/debug_util.h"
-#include "util/disk_info.h"
-#include "util/logging.h"
-#include "util/mem_info.h"
-#include "util/network_util.h"
-#include "util/thrift_util.h"
-#include "util/doris_metrics.h"
-#include "runtime/bufferpool/buffer_pool.h"
-#include "runtime/exec_env.h"
-#include "runtime/memory/chunk_allocator.h"
-#include "runtime/mem_tracker.h"
-#include "runtime/user_function_cache.h"
-#include "exprs/operators.h"
-#include "exprs/is_null_predicate.h"
-#include "exprs/like_predicate.h"
-#include "exprs/compound_predicate.h"
-#include "exprs/new_in_predicate.h"
-#include "exprs/string_functions.h"
+#include "exprs/bitmap_function.h"
#include "exprs/cast_functions.h"
-#include "exprs/math_functions.h"
+#include "exprs/compound_predicate.h"
+#include "exprs/decimal_operators.h"
+#include "exprs/decimalv2_operators.h"
#include "exprs/encryption_functions.h"
#include "exprs/es_functions.h"
+#include "exprs/grouping_sets_functions.h"
#include "exprs/hash_functions.h"
-#include "exprs/timestamp_functions.h"
-#include "exprs/decimal_operators.h"
-#include "exprs/decimalv2_operators.h"
+#include "exprs/hll_function.h"
+#include "exprs/hll_hash_function.h"
+#include "exprs/is_null_predicate.h"
+#include "exprs/json_functions.h"
+#include "exprs/like_predicate.h"
+#include "exprs/math_functions.h"
+#include "exprs/new_in_predicate.h"
+#include "exprs/operators.h"
+#include "exprs/string_functions.h"
#include "exprs/time_operators.h"
+#include "exprs/timestamp_functions.h"
#include "exprs/utility_functions.h"
-#include "exprs/json_functions.h"
-#include "exprs/hll_hash_function.h"
-#include "exprs/grouping_sets_functions.h"
-#include "exprs/bitmap_function.h"
-#include "exprs/hll_function.h"
#include "geo/geo_functions.h"
#include "olap/options.h"
-#include "util/time.h"
+#include "runtime/bufferpool/buffer_pool.h"
+#include "runtime/exec_env.h"
+#include "runtime/mem_tracker.h"
+#include "runtime/memory/chunk_allocator.h"
+#include "runtime/user_function_cache.h"
+#include "util/cpu_info.h"
+#include "util/debug_util.h"
+#include "util/disk_info.h"
+#include "util/doris_metrics.h"
+#include "util/logging.h"
+#include "util/mem_info.h"
+#include "util/network_util.h"
#include "util/system_metrics.h"
+#include "util/thrift_util.h"
+#include "util/time.h"
namespace doris {
@@ -69,7 +69,8 @@ void Daemon::tcmalloc_gc_thread() {
size_t used_size = 0;
size_t free_size = 0;
- MallocExtension::instance()->GetNumericProperty("generic.current_allocated_bytes", &used_size);
+ MallocExtension::instance()->GetNumericProperty("generic.current_allocated_bytes",
+ &used_size);
MallocExtension::instance()->GetNumericProperty("tcmalloc.pageheap_free_bytes", &free_size);
size_t alloc_size = used_size + free_size;
@@ -83,7 +84,8 @@ void Daemon::tcmalloc_gc_thread() {
}
void Daemon::memory_maintenance_thread() {
- while (!_stop_background_threads_latch.wait_for(MonoDelta::FromSeconds(config::memory_maintenance_sleep_time_s))) {
+ while (!_stop_background_threads_latch.wait_for(
+ MonoDelta::FromSeconds(config::memory_maintenance_sleep_time_s))) {
ExecEnv* env = ExecEnv::GetInstance();
// ExecEnv may not have been created yet or this may be the catalogd or statestored,
// which don't have ExecEnvs.
@@ -98,7 +100,7 @@ void Daemon::memory_maintenance_thread() {
// if the system is idle, we need to refresh the tracker occasionally since
// untracked memory may be allocated or freed, e.g. by background threads.
if (env->process_mem_tracker() != nullptr &&
- !env->process_mem_tracker()->is_consumption_metric_null()) {
+ !env->process_mem_tracker()->is_consumption_metric_null()) {
env->process_mem_tracker()->RefreshConsumptionFromMetric();
}
}
@@ -130,29 +132,31 @@ void Daemon::calculate_metrics_thread() {
lst_push_bytes = DorisMetrics::instance()->push_request_write_bytes->value();
lst_query_bytes = DorisMetrics::instance()->query_scan_bytes->value();
DorisMetrics::instance()->system_metrics()->get_disks_io_time(&lst_disks_io_time);
- DorisMetrics::instance()->system_metrics()->get_network_traffic(&lst_net_send_bytes, &lst_net_receive_bytes);
+ DorisMetrics::instance()->system_metrics()->get_network_traffic(&lst_net_send_bytes,
+ &lst_net_receive_bytes);
} else {
int64_t current_ts = GetCurrentTimeMicros() / 1000;
long interval = (current_ts - last_ts) / 1000;
last_ts = current_ts;
// 1. push bytes per second
- int64_t current_push_bytes = DorisMetrics::instance()->push_request_write_bytes->value();
+ int64_t current_push_bytes =
+ DorisMetrics::instance()->push_request_write_bytes->value();
int64_t pps = (current_push_bytes - lst_push_bytes) / (interval + 1);
- DorisMetrics::instance()->push_request_write_bytes_per_second->set_value(
- pps < 0 ? 0 : pps);
+ DorisMetrics::instance()->push_request_write_bytes_per_second->set_value(pps < 0 ? 0
+ : pps);
lst_push_bytes = current_push_bytes;
// 2. query bytes per second
int64_t current_query_bytes = DorisMetrics::instance()->query_scan_bytes->value();
int64_t qps = (current_query_bytes - lst_query_bytes) / (interval + 1);
- DorisMetrics::instance()->query_scan_bytes_per_second->set_value(
- qps < 0 ? 0 : qps);
+ DorisMetrics::instance()->query_scan_bytes_per_second->set_value(qps < 0 ? 0 : qps);
lst_query_bytes = current_query_bytes;
// 3. max disk io util
DorisMetrics::instance()->max_disk_io_util_percent->set_value(
- DorisMetrics::instance()->system_metrics()->get_max_io_util(lst_disks_io_time, 15));
+ DorisMetrics::instance()->system_metrics()->get_max_io_util(lst_disks_io_time,
+ 15));
// update lst map
DorisMetrics::instance()->system_metrics()->get_disks_io_time(&lst_disks_io_time);
@@ -160,11 +164,12 @@ void Daemon::calculate_metrics_thread() {
int64_t max_send = 0;
int64_t max_receive = 0;
DorisMetrics::instance()->system_metrics()->get_max_net_traffic(
- lst_net_send_bytes, lst_net_receive_bytes, 15, &max_send, &max_receive);
+ lst_net_send_bytes, lst_net_receive_bytes, 15, &max_send, &max_receive);
DorisMetrics::instance()->max_network_send_bytes_rate->set_value(max_send);
DorisMetrics::instance()->max_network_receive_bytes_rate->set_value(max_receive);
// update lst map
- DorisMetrics::instance()->system_metrics()->get_network_traffic(&lst_net_send_bytes, &lst_net_receive_bytes);
+ DorisMetrics::instance()->system_metrics()->get_network_traffic(&lst_net_send_bytes,
+ &lst_net_receive_bytes);
}
} while (!_stop_background_threads_latch.wait_for(MonoDelta::FromSeconds(15)));
}
@@ -189,15 +194,14 @@ static void init_doris_metrics(const std::vector<StorePath>& store_paths) {
return;
}
}
- DorisMetrics::instance()->initialize(
- init_system_metrics, disk_devices, network_interfaces);
+ DorisMetrics::instance()->initialize(init_system_metrics, disk_devices, network_interfaces);
}
void sigterm_handler(int signo) {
k_doris_exit = true;
}
-int install_signal(int signo, void(*handler)(int)) {
+int install_signal(int signo, void (*handler)(int)) {
struct sigaction sa;
memset(&sa, 0, sizeof(struct sigaction));
sa.sa_handler = handler;
@@ -205,9 +209,8 @@ int install_signal(int signo, void(*handler)(int)) {
auto ret = sigaction(signo, &sa, nullptr);
if (ret != 0) {
char buf[64];
- LOG(ERROR) << "install signal failed, signo=" << signo
- << ", errno=" << errno
- << ", errmsg=" << strerror_r(errno, buf, sizeof(buf));
+ LOG(ERROR) << "install signal failed, signo=" << signo << ", errno=" << errno
+ << ", errmsg=" << strerror_r(errno, buf, sizeof(buf));
}
return ret;
}
@@ -272,20 +275,20 @@ void Daemon::init(int argc, char** argv, const std::vector<StorePath>& paths) {
void Daemon::start() {
Status st;
#if !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && !defined(THREAD_SANITIZER)
- st = Thread::create("Daemon", "tcmalloc_gc_thread",
- [this]() { this->tcmalloc_gc_thread(); },
- &_tcmalloc_gc_thread);
+ st = Thread::create(
+ "Daemon", "tcmalloc_gc_thread", [this]() { this->tcmalloc_gc_thread(); },
+ &_tcmalloc_gc_thread);
CHECK(st.ok()) << st.to_string();
#endif
- st = Thread::create("Daemon", "memory_maintenance_thread",
- [this]() { this->memory_maintenance_thread(); },
- &_memory_maintenance_thread);
+ st = Thread::create(
+ "Daemon", "memory_maintenance_thread", [this]() { this->memory_maintenance_thread(); },
+ &_memory_maintenance_thread);
CHECK(st.ok()) << st.to_string();
if (config::enable_metric_calculator) {
- st = Thread::create("Daemon", "calculate_metrics_thread",
- [this]() { this->calculate_metrics_thread(); },
- &_calculate_metrics_thread);
+ st = Thread::create(
+ "Daemon", "calculate_metrics_thread",
+ [this]() { this->calculate_metrics_thread(); }, &_calculate_metrics_thread);
CHECK(st.ok()) << st.to_string();
}
}
@@ -304,4 +307,4 @@ void Daemon::stop() {
}
}
-} // namespace doris
+} // namespace doris
diff --git a/be/src/common/daemon.h b/be/src/common/daemon.h
index c11d99f..4544c1a 100644
--- a/be/src/common/daemon.h
+++ b/be/src/common/daemon.h
@@ -54,4 +54,4 @@ private:
scoped_refptr<Thread> _memory_maintenance_thread;
scoped_refptr<Thread> _calculate_metrics_thread;
};
-} // namespace doris
+} // namespace doris
diff --git a/be/src/common/global_types.h b/be/src/common/global_types.h
index ebe2815..014c402 100644
--- a/be/src/common/global_types.h
+++ b/be/src/common/global_types.h
@@ -28,6 +28,6 @@ typedef int SlotId;
typedef int TableId;
typedef int PlanNodeId;
-};
+}; // namespace doris
#endif
diff --git a/be/src/common/hdfs.h b/be/src/common/hdfs.h
index f87d0be..f41fc4a 100644
--- a/be/src/common/hdfs.h
+++ b/be/src/common/hdfs.h
@@ -29,4 +29,3 @@ typedef void* hdfsFile;
#endif
#endif
-
diff --git a/be/src/common/logconfig.cpp b/be/src/common/logconfig.cpp
index 72d3b7b..e3463f1 100644
--- a/be/src/common/logconfig.cpp
+++ b/be/src/common/logconfig.cpp
@@ -15,19 +15,18 @@
// specific language governing permissions and limitations
// under the License.
-#include "util/logging.h"
+#include <glog/logging.h>
+#include <glog/vlog_is_on.h>
-#include <iostream>
#include <cerrno>
-#include <cstring>
#include <cstdlib>
+#include <cstring>
+#include <iostream>
#include <mutex>
-#include <glog/logging.h>
-#include <glog/vlog_is_on.h>
-
#include "common/config.h"
#include "gutil/stringprintf.h"
+#include "util/logging.h"
namespace doris {
@@ -35,8 +34,7 @@ static bool logging_initialized = false;
static std::mutex logging_mutex;
-static bool iequals(const std::string& a, const std::string& b)
-{
+static bool iequals(const std::string& a, const std::string& b) {
unsigned int sz = a.size();
if (b.size() != sz) {
return false;
@@ -47,10 +45,9 @@ static bool iequals(const std::string& a, const std::string& b)
}
}
return true;
-}
+}
bool init_glog(const char* basename, bool install_signal_handler) {
-
std::lock_guard<std::mutex> logging_lock(logging_mutex);
if (logging_initialized) {
@@ -68,10 +65,10 @@ bool init_glog(const char* basename, bool install_signal_handler) {
// 0 means buffer INFO only
FLAGS_logbuflevel = 0;
// buffer log messages for at most this many seconds
- FLAGS_logbufsecs = 30;
+ FLAGS_logbufsecs = 30;
// set roll num
FLAGS_log_filenum_quota = config::sys_log_roll_num;
-
+
// set log level
std::string& loglevel = config::sys_log_level;
if (iequals(loglevel, "INFO")) {
@@ -109,7 +106,7 @@ bool init_glog(const char* basename, bool install_signal_handler) {
} else if (rollmode.substr(0, sizeflag.length()).compare(sizeflag) == 0) {
FLAGS_log_split_method = "size";
std::string sizestr = rollmode.substr(sizeflag.size(), rollmode.size() - sizeflag.size());
- if (sizestr.size() != 0) {
+ if (sizestr.size() != 0) {
char* end = NULL;
errno = 0;
const char* sizecstr = sizestr.c_str();
@@ -143,9 +140,8 @@ bool init_glog(const char* basename, bool install_signal_handler) {
google::InitGoogleLogging(basename);
logging_initialized = true;
-
- return true;
+ return true;
}
void shutdown_logging() {
@@ -159,13 +155,8 @@ std::string FormatTimestampForLog(MicrosecondsInt64 micros_since_epoch) {
struct tm tm_time;
localtime_r(&secs_since_epoch, &tm_time);
- return StringPrintf("%02d%02d %02d:%02d:%02d.%06d",
- 1 + tm_time.tm_mon,
- tm_time.tm_mday,
- tm_time.tm_hour,
- tm_time.tm_min,
- tm_time.tm_sec,
- usecs);
+ return StringPrintf("%02d%02d %02d:%02d:%02d.%06d", 1 + tm_time.tm_mon, tm_time.tm_mday,
+ tm_time.tm_hour, tm_time.tm_min, tm_time.tm_sec, usecs);
}
} // namespace doris
diff --git a/be/src/common/logging.h b/be/src/common/logging.h
index 14545bc..e3c3130 100644
--- a/be/src/common/logging.h
+++ b/be/src/common/logging.h
@@ -23,16 +23,25 @@
// issues when we try to dynamically link the codegen'd functions.
#ifdef IR_COMPILE
#include <iostream>
-#define DCHECK(condition) while (false) std::cout
-#define DCHECK_EQ(a, b) while(false) std::cout
-#define DCHECK_NE(a, b) while(false) std::cout
-#define DCHECK_GT(a, b) while(false) std::cout
-#define DCHECK_LT(a, b) while(false) std::cout
-#define DCHECK_GE(a, b) while(false) std::cout
-#define DCHECK_LE(a, b) while(false) std::cout
+#define DCHECK(condition) \
+ while (false) std::cout
+#define DCHECK_EQ(a, b) \
+ while (false) std::cout
+#define DCHECK_NE(a, b) \
+ while (false) std::cout
+#define DCHECK_GT(a, b) \
+ while (false) std::cout
+#define DCHECK_LT(a, b) \
+ while (false) std::cout
+#define DCHECK_GE(a, b) \
+ while (false) std::cout
+#define DCHECK_LE(a, b) \
+ while (false) std::cout
// Similar to how glog defines DCHECK for release.
-#define LOG(level) while (false) std::cout
-#define VLOG(level) while (false) std::cout
+#define LOG(level) \
+ while (false) std::cout
+#define VLOG(level) \
+ while (false) std::cout
#else
// GLOG defines this based on the system but doesn't check if it's already
// been defined. undef it first to avoid warnings.
@@ -48,11 +57,11 @@
// Define VLOG levels. We want display per-row info less than per-file which
// is less than per-query. For now per-connection is the same as per-query.
#define VLOG_CONNECTION VLOG(1)
-#define VLOG_RPC VLOG(8)
-#define VLOG_QUERY VLOG(1)
-#define VLOG_FILE VLOG(2)
-#define VLOG_ROW VLOG(10)
-#define VLOG_PROGRESS VLOG(2)
+#define VLOG_RPC VLOG(8)
+#define VLOG_QUERY VLOG(1)
+#define VLOG_FILE VLOG(2)
+#define VLOG_ROW VLOG(10)
+#define VLOG_PROGRESS VLOG(2)
#define VLOG_CONNECTION_IS_ON VLOG_IS_ON(1)
#define VLOG_RPC_IS_ON VLOG_IS_ON(2)
@@ -63,8 +72,8 @@
/// Define a wrapper around DCHECK for strongly typed enums that print a useful error
/// message on failure.
-#define DCHECK_ENUM_EQ(a, b) \
- DCHECK(a == b) << "[ " #a " = " << static_cast<int>(a) << " , " #b " = " \
- << static_cast<int>(b) << " ]"
+#define DCHECK_ENUM_EQ(a, b) \
+ DCHECK(a == b) << "[ " #a " = " << static_cast<int>(a) << " , " #b " = " \
+ << static_cast<int>(b) << " ]"
#endif
diff --git a/be/src/common/names.h b/be/src/common/names.h
old mode 100755
new mode 100644
index c0acdcd..890b562
--- a/be/src/common/names.h
+++ b/be/src/common/names.h
@@ -77,14 +77,12 @@ using std::setfill;
using std::setw;
#endif
-
#ifdef _GLIBCXX_FSTREAM
using std::fstream;
using std::ifstream;
using std::ofstream;
#endif
-
#ifdef _GLIBCXX_SSTREAM
using std::stringstream;
using std::istringstream;
@@ -133,14 +131,14 @@ using boost::lexical_cast;
using boost::shared_mutex;
#endif
-
/// In older versions of boost, when including mutex.hpp, it would include locks.hpp that
/// would in turn provide lock_guard<>. In more recent versions, including mutex.hpp would
/// include lock_types.hpp that does not provide lock_guard<>. This check verifies if boost
/// locks have been included and makes sure to only include lock_guard if the provided lock
/// implementations were not included using lock_types.hpp (for older boost versions) or if
/// lock_guard.hpp was explicitly included.
-#if (defined(BOOST_THREAD_LOCKS_HPP) && BOOST_VERSION < 105300) || defined(BOOST_THREAD_LOCK_GUARD_HPP)
+#if (defined(BOOST_THREAD_LOCKS_HPP) && BOOST_VERSION < 105300) || \
+ defined(BOOST_THREAD_LOCK_GUARD_HPP)
using boost::lock_guard;
#endif
diff --git a/be/src/common/object_pool.h b/be/src/common/object_pool.h
index 70d52e8..054a410 100644
--- a/be/src/common/object_pool.h
+++ b/be/src/common/object_pool.h
@@ -18,9 +18,9 @@
#ifndef DORIS_BE_SRC_COMMON_COMMON_OBJECT_POOL_H
#define DORIS_BE_SRC_COMMON_COMMON_OBJECT_POOL_H
-#include <vector>
-#include <boost/thread/mutex.hpp>
#include <boost/thread/locks.hpp>
+#include <boost/thread/mutex.hpp>
+#include <vector>
#include "util/spinlock.h"
@@ -31,11 +31,9 @@ namespace doris {
// Thread-safe.
class ObjectPool {
public:
- ObjectPool(): _objects() {}
+ ObjectPool() : _objects() {}
- ~ObjectPool() {
- clear();
- }
+ ~ObjectPool() { clear(); }
template <class T>
T* add(T* t) {
@@ -70,10 +68,8 @@ private:
template <class T>
struct SpecificElement : GenericElement {
- SpecificElement(T* t): t(t) {}
- ~SpecificElement() {
- delete t;
- }
+ SpecificElement(T* t) : t(t) {}
+ ~SpecificElement() { delete t; }
T* t;
};
@@ -83,6 +79,6 @@ private:
SpinLock _lock;
};
-}
+} // namespace doris
#endif
diff --git a/be/src/common/resource_tls.cpp b/be/src/common/resource_tls.cpp
index ddf916d..b5fd7af 100644
--- a/be/src/common/resource_tls.cpp
+++ b/be/src/common/resource_tls.cpp
@@ -64,4 +64,4 @@ int ResourceTls::set_resource_tls(TResourceInfo* info) {
return ret;
}
-}
+} // namespace doris
diff --git a/be/src/common/resource_tls.h b/be/src/common/resource_tls.h
index 0a51f4a..ed72023 100644
--- a/be/src/common/resource_tls.h
+++ b/be/src/common/resource_tls.h
@@ -28,6 +28,6 @@ public:
static int set_resource_tls(TResourceInfo*);
};
-}
+} // namespace doris
#endif
diff --git a/be/src/common/status.cpp b/be/src/common/status.cpp
index 446ae5a..75eab87 100644
--- a/be/src/common/status.cpp
+++ b/be/src/common/status.cpp
@@ -8,8 +8,8 @@
namespace doris {
-inline const char* assemble_state(
- TStatusCode::type code, const Slice& msg, int16_t precise_code, const Slice& msg2) {
+inline const char* assemble_state(TStatusCode::type code, const Slice& msg, int16_t precise_code,
+ const Slice& msg2) {
DCHECK(code != TStatusCode::OK);
const uint32_t len1 = msg.size;
@@ -58,8 +58,7 @@ Status::Status(const PStatus& s) : _state(nullptr) {
}
Status::Status(TStatusCode::type code, const Slice& msg, int16_t precise_code, const Slice& msg2)
- : _state(assemble_state(code, msg, precise_code, msg2)) {
-}
+ : _state(assemble_state(code, msg, precise_code, msg2)) {}
void Status::to_thrift(TStatus* s) const {
s->error_msgs.clear();
@@ -196,4 +195,4 @@ Status Status::clone_and_append(const Slice& msg) const {
return Status(code(), message(), precise_code(), msg);
}
-}
+} // namespace doris
diff --git a/be/src/common/status.h b/be/src/common/status.h
index 8fc755f..9f95472 100644
--- a/be/src/common/status.h
+++ b/be/src/common/status.h
@@ -7,23 +7,21 @@
#include <string>
#include <vector>
-#include "common/logging.h"
#include "common/compiler_util.h"
-#include "gen_cpp/Status_types.h" // for TStatus
-#include "gen_cpp/status.pb.h" // for PStatus
-#include "util/slice.h" // for Slice
+#include "common/logging.h"
+#include "gen_cpp/Status_types.h" // for TStatus
+#include "gen_cpp/status.pb.h" // for PStatus
+#include "util/slice.h" // for Slice
namespace doris {
class Status {
public:
- Status(): _state(nullptr) {}
+ Status() : _state(nullptr) {}
~Status() noexcept { delete[] _state; }
// copy c'tor makes copy of error detail so Status can be returned by value
- Status(const Status& s)
- : _state(s._state == nullptr ? nullptr : copy_state(s._state)) {
- }
+ Status(const Status& s) : _state(s._state == nullptr ? nullptr : copy_state(s._state)) {}
// same as copy c'tor
Status& operator=(const Status& s) {
@@ -37,9 +35,7 @@ public:
}
// move c'tor
- Status(Status&& s) noexcept : _state(s._state) {
- s._state = nullptr;
- }
+ Status(Status&& s) noexcept : _state(s._state) { s._state = nullptr; }
// move assign
Status& operator=(Status&& s) noexcept {
@@ -54,97 +50,96 @@ public:
static Status OK() { return Status(); }
- static Status PublishTimeout(const Slice& msg, int16_t precise_code = 1, const Slice& msg2 = Slice()) {
+ static Status PublishTimeout(const Slice& msg, int16_t precise_code = 1,
+ const Slice& msg2 = Slice()) {
return Status(TStatusCode::PUBLISH_TIMEOUT, msg, precise_code, msg2);
}
- static Status MemoryAllocFailed(const Slice& msg, int16_t precise_code = 1, const Slice& msg2 = Slice()) {
+ static Status MemoryAllocFailed(const Slice& msg, int16_t precise_code = 1,
+ const Slice& msg2 = Slice()) {
return Status(TStatusCode::MEM_ALLOC_FAILED, msg, precise_code, msg2);
}
- static Status BufferAllocFailed(const Slice& msg, int16_t precise_code = 1, const Slice& msg2 = Slice()) {
+ static Status BufferAllocFailed(const Slice& msg, int16_t precise_code = 1,
+ const Slice& msg2 = Slice()) {
return Status(TStatusCode::BUFFER_ALLOCATION_FAILED, msg, precise_code, msg2);
}
- static Status InvalidArgument(const Slice& msg, int16_t precise_code = 1, const Slice& msg2 = Slice()) {
+ static Status InvalidArgument(const Slice& msg, int16_t precise_code = 1,
+ const Slice& msg2 = Slice()) {
return Status(TStatusCode::INVALID_ARGUMENT, msg, precise_code, msg2);
}
- static Status MinimumReservationUnavailable(const Slice& msg, int16_t precise_code = 1, const Slice& msg2 = Slice()) {
+ static Status MinimumReservationUnavailable(const Slice& msg, int16_t precise_code = 1,
+ const Slice& msg2 = Slice()) {
return Status(TStatusCode::MINIMUM_RESERVATION_UNAVAILABLE, msg, precise_code, msg2);
}
- static Status Corruption(const Slice& msg, int16_t precise_code = 1, const Slice& msg2 = Slice()) {
+ static Status Corruption(const Slice& msg, int16_t precise_code = 1,
+ const Slice& msg2 = Slice()) {
return Status(TStatusCode::CORRUPTION, msg, precise_code, msg2);
}
- static Status IOError(const Slice& msg,
- int16_t precise_code = 1,
- const Slice& msg2 = Slice()) {
+ static Status IOError(const Slice& msg, int16_t precise_code = 1, const Slice& msg2 = Slice()) {
return Status(TStatusCode::IO_ERROR, msg, precise_code, msg2);
}
- static Status NotFound(const Slice& msg,
- int16_t precise_code = 1,
- const Slice& msg2 = Slice()) {
+ static Status NotFound(const Slice& msg, int16_t precise_code = 1,
+ const Slice& msg2 = Slice()) {
return Status(TStatusCode::NOT_FOUND, msg, precise_code, msg2);
}
- static Status AlreadyExist(const Slice& msg,
- int16_t precise_code = 1,
+ static Status AlreadyExist(const Slice& msg, int16_t precise_code = 1,
const Slice& msg2 = Slice()) {
return Status(TStatusCode::ALREADY_EXIST, msg, precise_code, msg2);
}
- static Status NotSupported(const Slice& msg,
- int16_t precise_code = 1,
+ static Status NotSupported(const Slice& msg, int16_t precise_code = 1,
const Slice& msg2 = Slice()) {
return Status(TStatusCode::NOT_IMPLEMENTED_ERROR, msg, precise_code, msg2);
}
- static Status EndOfFile(const Slice& msg,
- int16_t precise_code = 1,
+ static Status EndOfFile(const Slice& msg, int16_t precise_code = 1,
const Slice& msg2 = Slice()) {
return Status(TStatusCode::END_OF_FILE, msg, precise_code, msg2);
}
- static Status InternalError(const Slice& msg,
- int16_t precise_code = 1,
- const Slice& msg2 = Slice()) {
+ static Status InternalError(const Slice& msg, int16_t precise_code = 1,
+ const Slice& msg2 = Slice()) {
return Status(TStatusCode::INTERNAL_ERROR, msg, precise_code, msg2);
}
- static Status RuntimeError(const Slice& msg,
- int16_t precise_code = 1,
+ static Status RuntimeError(const Slice& msg, int16_t precise_code = 1,
const Slice& msg2 = Slice()) {
return Status(TStatusCode::RUNTIME_ERROR, msg, precise_code, msg2);
}
- static Status Cancelled(const Slice& msg, int16_t precise_code = 1, const Slice& msg2 = Slice()) {
+ static Status Cancelled(const Slice& msg, int16_t precise_code = 1,
+ const Slice& msg2 = Slice()) {
return Status(TStatusCode::CANCELLED, msg, precise_code, msg2);
}
- static Status MemoryLimitExceeded(const Slice& msg, int16_t precise_code = 1, const Slice& msg2 = Slice()) {
+ static Status MemoryLimitExceeded(const Slice& msg, int16_t precise_code = 1,
+ const Slice& msg2 = Slice()) {
return Status(TStatusCode::MEM_LIMIT_EXCEEDED, msg, precise_code, msg2);
}
- static Status ThriftRpcError(const Slice& msg, int16_t precise_code = 1, const Slice& msg2 = Slice()) {
+ static Status ThriftRpcError(const Slice& msg, int16_t precise_code = 1,
+ const Slice& msg2 = Slice()) {
return Status(TStatusCode::THRIFT_RPC_ERROR, msg, precise_code, msg2);
}
- static Status TimedOut(const Slice& msg, int16_t precise_code = 1, const Slice& msg2 = Slice()) {
+ static Status TimedOut(const Slice& msg, int16_t precise_code = 1,
+ const Slice& msg2 = Slice()) {
return Status(TStatusCode::TIMEOUT, msg, precise_code, msg2);
}
- static Status TooManyTasks(const Slice& msg, int16_t precise_code = 1, const Slice& msg2 = Slice()) {
+ static Status TooManyTasks(const Slice& msg, int16_t precise_code = 1,
+ const Slice& msg2 = Slice()) {
return Status(TStatusCode::TOO_MANY_TASKS, msg, precise_code, msg2);
}
- static Status ServiceUnavailable(const Slice& msg,
- int16_t precise_code = -1,
+ static Status ServiceUnavailable(const Slice& msg, int16_t precise_code = -1,
const Slice& msg2 = Slice()) {
return Status(TStatusCode::SERVICE_UNAVAILABLE, msg, precise_code, msg2);
}
- static Status Uninitialized(const Slice& msg,
- int16_t precise_code = -1,
+ static Status Uninitialized(const Slice& msg, int16_t precise_code = -1,
const Slice& msg2 = Slice()) {
return Status(TStatusCode::UNINITIALIZED, msg, precise_code, msg2);
}
- static Status Aborted(const Slice& msg,
- int16_t precise_code = -1,
+ static Status Aborted(const Slice& msg, int16_t precise_code = -1,
const Slice& msg2 = Slice()) {
return Status(TStatusCode::ABORTED, msg, precise_code, msg2);
}
- static Status DataQualityError(const Slice& msg,
- int16_t precise_code = -1,
- const Slice& msg2 = Slice()) {
+ static Status DataQualityError(const Slice& msg, int16_t precise_code = -1,
+ const Slice& msg2 = Slice()) {
return Status(TStatusCode::DATA_QUALITY_ERROR, msg, precise_code, msg2);
}
@@ -156,7 +151,7 @@ public:
bool is_end_of_file() const { return code() == TStatusCode::END_OF_FILE; }
bool is_not_found() const { return code() == TStatusCode::NOT_FOUND; }
bool is_already_exist() const { return code() == TStatusCode::ALREADY_EXIST; }
- bool is_io_error() const {return code() == TStatusCode::IO_ERROR; }
+ bool is_io_error() const { return code() == TStatusCode::IO_ERROR; }
/// @return @c true iff the status indicates Uninitialized.
bool is_uninitialized() const { return code() == TStatusCode::UNINITIALIZED; }
@@ -256,58 +251,58 @@ private:
};
// some generally useful macros
-#define RETURN_IF_ERROR(stmt) \
- do { \
+#define RETURN_IF_ERROR(stmt) \
+ do { \
const Status& _status_ = (stmt); \
- if (UNLIKELY(!_status_.ok())) { \
- return _status_; \
- } \
+ if (UNLIKELY(!_status_.ok())) { \
+ return _status_; \
+ } \
} while (false)
#define RETURN_IF_STATUS_ERROR(status, stmt) \
- do { \
- status = (stmt); \
- if (UNLIKELY(!status.ok())) { \
- return; \
- } \
+ do { \
+ status = (stmt); \
+ if (UNLIKELY(!status.ok())) { \
+ return; \
+ } \
} while (false)
-#define EXIT_IF_ERROR(stmt) \
- do { \
- const Status& _status_ = (stmt); \
- if (UNLIKELY(!_status_.ok())) { \
+#define EXIT_IF_ERROR(stmt) \
+ do { \
+ const Status& _status_ = (stmt); \
+ if (UNLIKELY(!_status_.ok())) { \
string msg = _status_.get_error_msg(); \
- LOG(ERROR) << msg; \
- exit(1); \
- } \
+ LOG(ERROR) << msg; \
+ exit(1); \
+ } \
} while (false)
/// @brief Emit a warning if @c to_call returns a bad status.
-#define WARN_IF_ERROR(to_call, warning_prefix) \
- do { \
- const Status& _s = (to_call); \
- if (UNLIKELY(!_s.ok())) { \
- LOG(WARNING) << (warning_prefix) << ": " << _s.to_string(); \
- } \
+#define WARN_IF_ERROR(to_call, warning_prefix) \
+ do { \
+ const Status& _s = (to_call); \
+ if (UNLIKELY(!_s.ok())) { \
+ LOG(WARNING) << (warning_prefix) << ": " << _s.to_string(); \
+ } \
} while (0);
-#define RETURN_WITH_WARN_IF_ERROR(stmt, ret_code, warning_prefix) \
- do { \
- const Status& _s = (stmt); \
- if (UNLIKELY(!_s.ok())) { \
+#define RETURN_WITH_WARN_IF_ERROR(stmt, ret_code, warning_prefix) \
+ do { \
+ const Status& _s = (stmt); \
+ if (UNLIKELY(!_s.ok())) { \
LOG(WARNING) << (warning_prefix) << ", error: " << _s.to_string(); \
- return ret_code; \
- } \
+ return ret_code; \
+ } \
} while (0);
-#define RETURN_NOT_OK_STATUS_WITH_WARN(stmt, warning_prefix) \
- do { \
- const Status& _s = (stmt); \
- if (UNLIKELY(!_s.ok())) { \
- LOG(WARNING) << (warning_prefix) << ", error: " << _s.to_string(); \
- return _s; \
- } \
+#define RETURN_NOT_OK_STATUS_WITH_WARN(stmt, warning_prefix) \
+ do { \
+ const Status& _s = (stmt); \
+ if (UNLIKELY(!_s.ok())) { \
+ LOG(WARNING) << (warning_prefix) << ", error: " << _s.to_string(); \
+ return _s; \
+ } \
} while (0);
-} // namespace doris
+} // namespace doris
#define WARN_UNUSED_RESULT __attribute__((warn_unused_result))
diff --git a/be/src/common/utils.h b/be/src/common/utils.h
index 8376fe3..4798483 100644
--- a/be/src/common/utils.h
+++ b/be/src/common/utils.h
@@ -30,7 +30,7 @@ struct AuthInfo {
int64_t auth_code = -1;
};
-template<class T>
+template <class T>
void set_request_auth(T* req, const AuthInfo& auth) {
if (auth.auth_code != -1) {
// if auth_code is set, no need to set other info
@@ -49,4 +49,4 @@ void set_request_auth(T* req, const AuthInfo& auth) {
}
}
-}
+} // namespace doris
diff --git a/be/src/env/env.h b/be/src/env/env.h
index 18a1cc4..d021573 100644
--- a/be/src/env/env.h
+++ b/be/src/env/env.h
@@ -35,12 +35,7 @@ public:
// CREATE_OR_OPEN | opens | creates
// MUST_CREATE | fails | creates
// MUST_EXIST | opens | fails
- enum OpenMode {
- CREATE_OR_OPEN_WITH_TRUNCATE,
- CREATE_OR_OPEN,
- MUST_CREATE,
- MUST_EXIST
- };
+ enum OpenMode { CREATE_OR_OPEN_WITH_TRUNCATE, CREATE_OR_OPEN, MUST_CREATE, MUST_EXIST };
Env() {}
virtual ~Env() {}
@@ -265,10 +260,7 @@ public:
// one of Append or PositionedAppend. We support only Append here.
class WritableFile {
public:
- enum FlushMode {
- FLUSH_SYNC,
- FLUSH_ASYNC
- };
+ enum FlushMode { FLUSH_SYNC, FLUSH_ASYNC };
WritableFile() {}
virtual ~WritableFile() {}
diff --git a/be/src/env/env_posix.cpp b/be/src/env/env_posix.cpp
index dd9bdc2..0ccdcb3 100644
--- a/be/src/env/env_posix.cpp
+++ b/be/src/env/env_posix.cpp
@@ -7,25 +7,25 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors
-#include "env/env.h"
-
#include <dirent.h>
#include <fcntl.h>
#include <stdio.h>
#include <sys/stat.h>
#include <sys/types.h>
-#include <unistd.h>
#include <sys/uio.h>
+#include <unistd.h>
+
#include <memory>
#include "common/logging.h"
+#include "env/env.h"
+#include "gutil/gscoped_ptr.h"
#include "gutil/macros.h"
#include "gutil/port.h"
-#include "gutil/gscoped_ptr.h"
#include "gutil/strings/substitute.h"
#include "util/errno.h"
-#include "util/slice.h"
#include "util/file_cache.h"
+#include "util/slice.h"
namespace doris {
@@ -101,8 +101,8 @@ static Status do_open(const string& filename, Env::OpenMode mode, int* fd) {
return Status::OK();
}
-static Status do_readv_at(int fd, const std::string& filename, uint64_t offset,
- const Slice* res, size_t res_cnt) {
+static Status do_readv_at(int fd, const std::string& filename, uint64_t offset, const Slice* res,
+ size_t res_cnt) {
// Convert the results into the iovec vector to request
// and calculate the total bytes requested
size_t bytes_req = 0;
@@ -110,7 +110,7 @@ static Status do_readv_at(int fd, const std::string& filename, uint64_t offset,
for (size_t i = 0; i < res_cnt; i++) {
const Slice& result = res[i];
bytes_req += result.size;
- iov[i] = { result.data, result.size };
+ iov[i] = {result.data, result.size};
}
uint64_t cur_offset = offset;
@@ -127,8 +127,8 @@ static Status do_readv_at(int fd, const std::string& filename, uint64_t offset,
}
if (PREDICT_FALSE(r == 0)) {
- return Status::EndOfFile(
- strings::Substitute("EOF trying to read $0 bytes at offset $1", bytes_req, offset));
+ return Status::EndOfFile(strings::Substitute("EOF trying to read $0 bytes at offset $1",
+ bytes_req, offset));
}
if (PREDICT_TRUE(r == rem)) {
@@ -146,7 +146,7 @@ static Status do_readv_at(int fd, const std::string& filename, uint64_t offset,
} else {
// Partially read this result.
// Adjust the iov_len and iov_base to request only the missing data.
- iov[i].iov_base = static_cast<uint8_t *>(iov[i].iov_base) + bytes_rem;
+ iov[i].iov_base = static_cast<uint8_t*>(iov[i].iov_base) + bytes_rem;
iov[i].iov_len -= bytes_rem;
break; // Don't need to adjust remaining iovec's
}
@@ -158,8 +158,8 @@ static Status do_readv_at(int fd, const std::string& filename, uint64_t offset,
return Status::OK();
}
-static Status do_writev_at(int fd, const string& filename, uint64_t offset,
- const Slice* data, size_t data_cnt, size_t* bytes_written) {
+static Status do_writev_at(int fd, const string& filename, uint64_t offset, const Slice* data,
+ size_t data_cnt, size_t* bytes_written) {
// Convert the results into the iovec vector to request
// and calculate the total bytes requested.
size_t bytes_req = 0;
@@ -167,7 +167,7 @@ static Status do_writev_at(int fd, const string& filename, uint64_t offset,
for (size_t i = 0; i < data_cnt; i++) {
const Slice& result = data[i];
bytes_req += result.size;
- iov[i] = { result.data, result.size };
+ iov[i] = {result.data, result.size};
}
uint64_t cur_offset = offset;
@@ -198,7 +198,7 @@ static Status do_writev_at(int fd, const string& filename, uint64_t offset,
} else {
// Partially wrote this result.
// Adjust the iov_len and iov_base to write only the missing data.
- iov[i].iov_base = static_cast<uint8_t *>(iov[i].iov_base) + bytes_rem;
+ iov[i].iov_base = static_cast<uint8_t*>(iov[i].iov_base) + bytes_rem;
iov[i].iov_len -= bytes_rem;
break; // Don't need to adjust remaining iovec's.
}
@@ -211,24 +211,22 @@ static Status do_writev_at(int fd, const string& filename, uint64_t offset,
return Status::OK();
}
-class PosixSequentialFile: public SequentialFile {
+class PosixSequentialFile : public SequentialFile {
public:
- PosixSequentialFile(string fname, FILE* f)
- : _filename(std::move(fname)), _file(f) {}
+ PosixSequentialFile(string fname, FILE* f) : _filename(std::move(fname)), _file(f) {}
~PosixSequentialFile() override {
int err;
RETRY_ON_EINTR(err, fclose(_file));
if (PREDICT_FALSE(err != 0)) {
LOG(WARNING) << "Failed to close " << _filename
- << ", msg=" << errno_to_string(ferror(_file));
+ << ", msg=" << errno_to_string(ferror(_file));
}
}
Status read(Slice* result) override {
size_t r;
- STREAM_RETRY_ON_EINTR(r, _file, fread_unlocked(result->data, 1,
- result->size, _file));
+ STREAM_RETRY_ON_EINTR(r, _file, fread_unlocked(result->data, 1, result->size, _file));
if (r < result->size) {
if (feof(_file)) {
// We leave status as ok if we hit the end of the file.
@@ -258,14 +256,13 @@ private:
class PosixRandomAccessFile : public RandomAccessFile {
public:
- PosixRandomAccessFile(std::string filename, int fd) : _filename(std::move(filename)), _fd(fd) {
- }
+ PosixRandomAccessFile(std::string filename, int fd) : _filename(std::move(filename)), _fd(fd) {}
~PosixRandomAccessFile() override {
int res;
RETRY_ON_EINTR(res, close(_fd));
if (res != 0) {
LOG(WARNING) << "close file failed, name=" << _filename
- << ", msg=" << errno_to_string(errno);
+ << ", msg=" << errno_to_string(errno);
}
}
@@ -287,6 +284,7 @@ public:
}
const std::string& file_name() const override { return _filename; }
+
private:
std::string _filename;
int _fd;
@@ -295,15 +293,16 @@ private:
class PosixWritableFile : public WritableFile {
public:
PosixWritableFile(std::string filename, int fd, uint64_t filesize, bool sync_on_close)
- : _filename(std::move(filename)), _fd(fd), _sync_on_close(sync_on_close), _filesize(filesize) { }
+ : _filename(std::move(filename)),
+ _fd(fd),
+ _sync_on_close(sync_on_close),
+ _filesize(filesize) {}
~PosixWritableFile() override {
WARN_IF_ERROR(close(), "Failed to close file, file=" + _filename);
}
- Status append(const Slice& data) override {
- return appendv(&data, 1);
- }
+ Status append(const Slice& data) override { return appendv(&data, 1); }
Status appendv(const Slice* data, size_t cnt) override {
size_t bytes_written = 0;
@@ -396,6 +395,7 @@ public:
uint64_t size() const override { return _filesize; }
const string& filename() const override { return _filename; }
+
private:
std::string _filename;
int _fd;
@@ -409,14 +409,9 @@ private:
class PosixRandomRWFile : public RandomRWFile {
public:
PosixRandomRWFile(string fname, int fd, bool sync_on_close)
- : _filename(std::move(fname)),
- _fd(fd),
- _sync_on_close(sync_on_close),
- _closed(false) {}
+ : _filename(std::move(fname)), _fd(fd), _sync_on_close(sync_on_close), _closed(false) {}
- ~PosixRandomRWFile() {
- WARN_IF_ERROR(close(), "Failed to close " + _filename);
- }
+ ~PosixRandomRWFile() { WARN_IF_ERROR(close(), "Failed to close " + _filename); }
virtual Status read_at(uint64_t offset, const Slice& result) const override {
return do_readv_at(_fd, _filename, offset, &result, 1);
@@ -452,9 +447,7 @@ public:
return Status::OK();
}
- Status sync() override {
- return do_sync(_fd, _filename);
- }
+ Status sync() override { return do_sync(_fd, _filename); }
Status close() override {
if (_closed) {
@@ -489,9 +482,7 @@ public:
return Status::OK();
}
- const string& filename() const override {
- return _filename;
- }
+ const string& filename() const override { return _filename; }
private:
const std::string _filename;
@@ -502,10 +493,10 @@ private:
class PosixEnv : public Env {
public:
- ~PosixEnv() override { }
+ ~PosixEnv() override {}
- Status new_sequential_file(
- const string& fname, std::unique_ptr<SequentialFile>* result) override {
+ Status new_sequential_file(const string& fname,
+ std::unique_ptr<SequentialFile>* result) override {
FILE* f;
POINTER_RETRY_ON_EINTR(f, fopen(fname.c_str(), "r"));
if (f == nullptr) {
@@ -517,13 +508,12 @@ public:
// get a RandomAccessFile pointer without file cache
Status new_random_access_file(const std::string& fname,
- std::unique_ptr<RandomAccessFile>* result) override {
+ std::unique_ptr<RandomAccessFile>* result) override {
return new_random_access_file(RandomAccessFileOptions(), fname, result);
}
- Status new_random_access_file(const RandomAccessFileOptions& opts,
- const std::string& fname,
- std::unique_ptr<RandomAccessFile>* result) override {
+ Status new_random_access_file(const RandomAccessFileOptions& opts, const std::string& fname,
+ std::unique_ptr<RandomAccessFile>* result) override {
int fd;
RETRY_ON_EINTR(fd, open(fname.c_str(), O_RDONLY));
if (fd < 0) {
@@ -533,13 +523,11 @@ public:
return Status::OK();
}
- Status new_writable_file(const string& fname,
- std::unique_ptr<WritableFile>* result) override {
+ Status new_writable_file(const string& fname, std::unique_ptr<WritableFile>* result) override {
return new_writable_file(WritableFileOptions(), fname, result);
}
- Status new_writable_file(const WritableFileOptions& opts,
- const string& fname,
+ Status new_writable_file(const WritableFileOptions& opts, const string& fname,
std::unique_ptr<WritableFile>* result) override {
int fd;
RETURN_IF_ERROR(do_open(fname, opts.mode, &fd));
@@ -552,13 +540,11 @@ public:
return Status::OK();
}
- Status new_random_rw_file(const string& fname,
- std::unique_ptr<RandomRWFile>* result) override {
+ Status new_random_rw_file(const string& fname, std::unique_ptr<RandomRWFile>* result) override {
return new_random_rw_file(RandomRWFileOptions(), fname, result);
}
- Status new_random_rw_file(const RandomRWFileOptions& opts,
- const string& fname,
+ Status new_random_rw_file(const RandomRWFileOptions& opts, const string& fname,
std::unique_ptr<RandomRWFile>* result) override {
int fd;
RETURN_IF_ERROR(do_open(fname, opts.mode, &fd));
@@ -573,8 +559,7 @@ public:
return Status::OK();
}
- Status get_children(const std::string& dir,
- std::vector<std::string>* result) override {
+ Status get_children(const std::string& dir, std::vector<std::string>* result) override {
result->clear();
DIR* d = opendir(dir.c_str());
if (d == nullptr) {
@@ -648,7 +633,7 @@ public:
Status sync_dir(const string& dirname) override {
int dir_fd;
- RETRY_ON_EINTR(dir_fd, open(dirname.c_str(), O_DIRECTORY|O_RDONLY));
+ RETRY_ON_EINTR(dir_fd, open(dirname.c_str(), O_DIRECTORY | O_RDONLY));
if (dir_fd < 0) {
return io_error(dirname, errno);
}
@@ -693,7 +678,7 @@ public:
Status get_file_modified_time(const std::string& fname, uint64_t* file_mtime) override {
struct stat s;
- if (stat(fname.c_str(), &s) !=0) {
+ if (stat(fname.c_str(), &s) != 0) {
return io_error(fname, errno);
}
*file_mtime = static_cast<uint64_t>(s.st_mtime);
diff --git a/be/src/env/env_util.cpp b/be/src/env/env_util.cpp
index 2c994e9..3494b2e 100644
--- a/be/src/env/env_util.cpp
+++ b/be/src/env/env_util.cpp
@@ -23,7 +23,6 @@
using std::shared_ptr;
using std::string;
-
namespace doris {
namespace env_util {
diff --git a/be/src/exec/aggregation_node.cpp b/be/src/exec/aggregation_node.cpp
index 53e3326..3edb36e 100644
--- a/be/src/exec/aggregation_node.cpp
+++ b/be/src/exec/aggregation_node.cpp
@@ -17,11 +17,12 @@
#include "exec/aggregation_node.h"
+#include <gperftools/profiler.h>
#include <math.h>
-#include <sstream>
-#include <boost/functional/hash.hpp>
#include <thrift/protocol/TDebugProtocol.h>
-#include <gperftools/profiler.h>
+
+#include <boost/functional/hash.hpp>
+#include <sstream>
#include "exec/hash_table.hpp"
#include "exprs/agg_fn_evaluator.h"
@@ -44,36 +45,33 @@ namespace doris {
// TODO: pass in maximum size; enforce by setting limit in mempool
// TODO: have a Status ExecNode::init(const TPlanNode&) member function
// that does initialization outside of c'tor, so we can indicate errors
-AggregationNode::AggregationNode(
- ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs) :
- ExecNode(pool, tnode, descs),
- _intermediate_tuple_id(tnode.agg_node.intermediate_tuple_id),
- _intermediate_tuple_desc(NULL),
- _output_tuple_id(tnode.agg_node.output_tuple_id),
- _output_tuple_desc(NULL),
- _singleton_output_tuple(NULL),
- //_tuple_pool(new MemPool()),
- //
- _process_row_batch_fn(NULL),
- _needs_finalize(tnode.agg_node.need_finalize),
- _build_timer(NULL),
- _get_results_timer(NULL),
- _hash_table_buckets_counter(NULL) {
-}
-
-AggregationNode::~AggregationNode() {
-}
+AggregationNode::AggregationNode(ObjectPool* pool, const TPlanNode& tnode,
+ const DescriptorTbl& descs)
+ : ExecNode(pool, tnode, descs),
+ _intermediate_tuple_id(tnode.agg_node.intermediate_tuple_id),
+ _intermediate_tuple_desc(NULL),
+ _output_tuple_id(tnode.agg_node.output_tuple_id),
+ _output_tuple_desc(NULL),
+ _singleton_output_tuple(NULL),
+ //_tuple_pool(new MemPool()),
+ //
+ _process_row_batch_fn(NULL),
+ _needs_finalize(tnode.agg_node.need_finalize),
+ _build_timer(NULL),
+ _get_results_timer(NULL),
+ _hash_table_buckets_counter(NULL) {}
+
+AggregationNode::~AggregationNode() {}
Status AggregationNode::init(const TPlanNode& tnode, RuntimeState* state) {
RETURN_IF_ERROR(ExecNode::init(tnode, state));
// ignore return status for now , so we need to introduce ExecNode::init()
- RETURN_IF_ERROR(Expr::create_expr_trees(
- _pool, tnode.agg_node.grouping_exprs, &_probe_expr_ctxs));
+ RETURN_IF_ERROR(
+ Expr::create_expr_trees(_pool, tnode.agg_node.grouping_exprs, &_probe_expr_ctxs));
for (int i = 0; i < tnode.agg_node.aggregate_functions.size(); ++i) {
AggFnEvaluator* evaluator = NULL;
- AggFnEvaluator::create(
- _pool, tnode.agg_node.aggregate_functions[i], &evaluator);
+ AggFnEvaluator::create(_pool, tnode.agg_node.aggregate_functions[i], &evaluator);
_aggregate_evaluators.push_back(evaluator);
}
return Status::OK();
@@ -83,19 +81,17 @@ Status AggregationNode::prepare(RuntimeState* state) {
RETURN_IF_ERROR(ExecNode::prepare(state));
_build_timer = ADD_TIMER(runtime_profile(), "BuildTime");
_get_results_timer = ADD_TIMER(runtime_profile(), "GetResultsTime");
- _hash_table_buckets_counter =
- ADD_COUNTER(runtime_profile(), "BuildBuckets", TUnit::UNIT);
+ _hash_table_buckets_counter = ADD_COUNTER(runtime_profile(), "BuildBuckets", TUnit::UNIT);
_hash_table_load_factor_counter =
- ADD_COUNTER(runtime_profile(), "LoadFactor", TUnit::DOUBLE_VALUE);
+ ADD_COUNTER(runtime_profile(), "LoadFactor", TUnit::DOUBLE_VALUE);
SCOPED_TIMER(_runtime_profile->total_time_counter());
- _intermediate_tuple_desc =
- state->desc_tbl().get_tuple_descriptor(_intermediate_tuple_id);
+ _intermediate_tuple_desc = state->desc_tbl().get_tuple_descriptor(_intermediate_tuple_id);
_output_tuple_desc = state->desc_tbl().get_tuple_descriptor(_output_tuple_id);
DCHECK_EQ(_intermediate_tuple_desc->slots().size(), _output_tuple_desc->slots().size());
- RETURN_IF_ERROR(Expr::prepare(
- _probe_expr_ctxs, state, child(0)->row_desc(), expr_mem_tracker()));
+ RETURN_IF_ERROR(
+ Expr::prepare(_probe_expr_ctxs, state, child(0)->row_desc(), expr_mem_tracker()));
// Construct build exprs from _agg_tuple_desc
for (int i = 0; i < _probe_expr_ctxs.size(); ++i) {
@@ -110,8 +106,7 @@ Status AggregationNode::prepare(RuntimeState* state) {
// nor this node's output row desc may contain the intermediate tuple, e.g.,
// in a single-node plan with an intermediate tuple different from the output tuple.
RowDescriptor build_row_desc(_intermediate_tuple_desc, false);
- RETURN_IF_ERROR(Expr::prepare(
- _build_expr_ctxs, state, build_row_desc, expr_mem_tracker()));
+ RETURN_IF_ERROR(Expr::prepare(_build_expr_ctxs, state, build_row_desc, expr_mem_tracker()));
_tuple_pool.reset(new MemPool(mem_tracker().get()));
@@ -134,9 +129,9 @@ Status AggregationNode::prepare(RuntimeState* state) {
}
// TODO: how many buckets?
- _hash_tbl.reset(new HashTable(
- _build_expr_ctxs, _probe_expr_ctxs, 1, true,
- std::vector<bool>(_build_expr_ctxs.size(), false), id(), mem_tracker(), 1024));
+ _hash_tbl.reset(new HashTable(_build_expr_ctxs, _probe_expr_ctxs, 1, true,
+ std::vector<bool>(_build_expr_ctxs.size(), false), id(),
+ mem_tracker(), 1024));
if (_probe_expr_ctxs.empty()) {
// create single output tuple now; we need to output something
@@ -177,8 +172,8 @@ Status AggregationNode::open(RuntimeState* state) {
if (VLOG_ROW_IS_ON) {
for (int i = 0; i < batch.num_rows(); ++i) {
TupleRow* row = batch.get_row(i);
- VLOG_ROW << "id=" << id() << " input row: "
- << row->to_string(_children[0]->row_desc());
+ VLOG_ROW << "id=" << id()
+ << " input row: " << row->to_string(_children[0]->row_desc());
}
}
@@ -223,7 +218,7 @@ Status AggregationNode::open(RuntimeState* state) {
}
VLOG_ROW << "id=" << id() << " aggregated " << num_input_rows << " input rows into "
- << num_agg_rows << " output rows";
+ << num_agg_rows << " output rows";
_output_iterator = _hash_tbl->begin();
return Status::OK();
}
@@ -255,8 +250,7 @@ Status AggregationNode::get_next(RuntimeState* state, RowBatch* row_batch, bool*
int row_idx = row_batch->add_row();
TupleRow* row = row_batch->get_row(row_idx);
Tuple* intermediate_tuple = _output_iterator.get_row()->get_tuple(0);
- Tuple* output_tuple =
- finalize_tuple(intermediate_tuple, row_batch->tuple_data_pool());
+ Tuple* output_tuple = finalize_tuple(intermediate_tuple, row_batch->tuple_data_pool());
row->set_tuple(0, output_tuple);
if (ExecNode::eval_conjuncts(ctxs, num_ctxs, row)) {
@@ -274,8 +268,7 @@ Status AggregationNode::get_next(RuntimeState* state, RowBatch* row_batch, bool*
*eos = _output_iterator.at_end() || reached_limit();
if (*eos) {
- if (_hash_tbl.get() != NULL &&
- _hash_table_buckets_counter != NULL) {
+ if (_hash_tbl.get() != NULL && _hash_table_buckets_counter != NULL) {
COUNTER_SET(_hash_table_buckets_counter, _hash_tbl->num_buckets());
}
}
@@ -328,7 +321,8 @@ Status AggregationNode::close(RuntimeState* state) {
Tuple* AggregationNode::construct_intermediate_tuple() {
Tuple* agg_tuple = Tuple::create(_intermediate_tuple_desc->byte_size(), _tuple_pool.get());
- std::vector<SlotDescriptor*>::const_iterator slot_desc = _intermediate_tuple_desc->slots().begin();
+ std::vector<SlotDescriptor*>::const_iterator slot_desc =
+ _intermediate_tuple_desc->slots().begin();
// copy grouping values
for (int i = 0; i < _probe_expr_ctxs.size(); ++i, ++slot_desc) {
@@ -359,8 +353,7 @@ Tuple* AggregationNode::construct_intermediate_tuple() {
// - max: min_value
// TODO: remove when we don't use the irbuilder for codegen here.
// This optimization no longer applies with AnyVal
- if (!(*slot_desc)->type().is_string_type() &&
- !(*slot_desc)->type().is_date_type()) {
+ if (!(*slot_desc)->type().is_string_type() && !(*slot_desc)->type().is_date_type()) {
ExprValue default_value;
void* default_value_ptr = NULL;
@@ -434,20 +427,18 @@ Tuple* AggregationNode::finalize_tuple(Tuple* tuple, MemPool* pool) {
void AggregationNode::debug_string(int indentation_level, std::stringstream* out) const {
*out << std::string(indentation_level * 2, ' ');
*out << "AggregationNode(intermediate_tuple_id=" << _intermediate_tuple_id
- << " output_tuple_id=" << _output_tuple_id
- << " needs_finalize=" << _needs_finalize
+ << " output_tuple_id=" << _output_tuple_id << " needs_finalize="
+ << _needs_finalize
// << " probe_exprs=" << Expr::debug_string(_probe_exprs)
<< " agg_exprs=" << AggFnEvaluator::debug_string(_aggregate_evaluators);
ExecNode::debug_string(indentation_level, out);
*out << ")";
}
-void AggregationNode::push_down_predicate(RuntimeState *state,
- std::list<ExprContext*> *expr_ctxs) {
+void AggregationNode::push_down_predicate(RuntimeState* state, std::list<ExprContext*>* expr_ctxs) {
// groupby can pushdown, agg can't pushdown
// Now we doesn't pushdown for easy.
return;
}
-}
-
+} // namespace doris
diff --git a/be/src/exec/aggregation_node.h b/be/src/exec/aggregation_node.h
index 72e9f0a..464d364 100644
--- a/be/src/exec/aggregation_node.h
+++ b/be/src/exec/aggregation_node.h
@@ -61,8 +61,7 @@ public:
virtual Status close(RuntimeState* state);
virtual void debug_string(int indentation_level, std::stringstream* out) const;
- virtual void push_down_predicate(
- RuntimeState *state, std::list<ExprContext*> *expr_ctxs);
+ virtual void push_down_predicate(RuntimeState* state, std::list<ExprContext*>* expr_ctxs);
private:
boost::scoped_ptr<HashTable> _hash_tbl;
@@ -73,7 +72,7 @@ private:
/// FunctionContext for each agg fn and backing pool.
std::vector<doris_udf::FunctionContext*> _agg_fn_ctxs;
boost::scoped_ptr<MemPool> _agg_fn_pool;
-
+
// Exprs used to evaluate input rows
std::vector<ExprContext*> _probe_expr_ctxs;
// Exprs used to insert constructed aggregation tuple into the hash table.
@@ -88,8 +87,8 @@ private:
/// the intermediate tuple.
TupleId _output_tuple_id;
TupleDescriptor* _output_tuple_desc;
-
- Tuple* _singleton_output_tuple; // result of aggregation w/o GROUP BY
+
+ Tuple* _singleton_output_tuple; // result of aggregation w/o GROUP BY
boost::scoped_ptr<MemPool> _tuple_pool;
typedef void (*ProcessRowBatchFn)(AggregationNode*, RowBatch*);
@@ -129,6 +128,6 @@ private:
void process_row_batch_with_grouping(RowBatch* batch, MemPool* pool);
};
-}
+} // namespace doris
#endif
diff --git a/be/src/exec/aggregation_node_ir.cpp b/be/src/exec/aggregation_node_ir.cpp
index 4ce5cf3..0d17441 100644
--- a/be/src/exec/aggregation_node_ir.cpp
+++ b/be/src/exec/aggregation_node_ir.cpp
@@ -16,7 +16,6 @@
// under the License.
#include "exec/aggregation_node.h"
-
#include "exec/hash_table.hpp"
#include "runtime/row_batch.h"
#include "runtime/runtime_state.h"
@@ -48,5 +47,4 @@ void AggregationNode::process_row_batch_with_grouping(RowBatch* batch, MemPool*
}
}
-}
-
+} // namespace doris
diff --git a/be/src/exec/analytic_eval_node.cpp b/be/src/exec/analytic_eval_node.cpp
index 6aae452..bb069a1 100644
--- a/be/src/exec/analytic_eval_node.cpp
+++ b/be/src/exec/analytic_eval_node.cpp
@@ -19,7 +19,6 @@
#include "exprs/agg_fn_evaluator.h"
#include "exprs/anyval_util.h"
-
#include "runtime/descriptors.h"
#include "runtime/row_batch.h"
#include "runtime/runtime_state.h"
@@ -30,41 +29,38 @@ namespace doris {
using doris_udf::BigIntVal;
AnalyticEvalNode::AnalyticEvalNode(ObjectPool* pool, const TPlanNode& tnode,
- const DescriptorTbl& descs) :
- ExecNode(pool, tnode, descs),
- _window(tnode.analytic_node.window),
- _intermediate_tuple_desc(
- descs.get_tuple_descriptor(tnode.analytic_node.intermediate_tuple_id)),
- _result_tuple_desc(
- descs.get_tuple_descriptor(tnode.analytic_node.output_tuple_id)),
- _buffered_tuple_desc(NULL),
- _partition_by_eq_expr_ctx(NULL),
- _order_by_eq_expr_ctx(NULL),
- _rows_start_offset(0),
- _rows_end_offset(0),
- _has_first_val_null_offset(false),
- _first_val_null_offset(0),
- _last_result_idx(-1),
- _prev_pool_last_result_idx(-1),
- _prev_pool_last_window_idx(-1),
- _curr_tuple(NULL),
- _dummy_result_tuple(NULL),
- _curr_partition_idx(-1),
- _prev_input_row(NULL),
- _block_mgr_client(nullptr),
- _input_eos(false),
- _evaluation_timer(NULL) {
+ const DescriptorTbl& descs)
+ : ExecNode(pool, tnode, descs),
+ _window(tnode.analytic_node.window),
+ _intermediate_tuple_desc(
+ descs.get_tuple_descriptor(tnode.analytic_node.intermediate_tuple_id)),
+ _result_tuple_desc(descs.get_tuple_descriptor(tnode.analytic_node.output_tuple_id)),
+ _buffered_tuple_desc(NULL),
+ _partition_by_eq_expr_ctx(NULL),
+ _order_by_eq_expr_ctx(NULL),
+ _rows_start_offset(0),
+ _rows_end_offset(0),
+ _has_first_val_null_offset(false),
+ _first_val_null_offset(0),
+ _last_result_idx(-1),
+ _prev_pool_last_result_idx(-1),
+ _prev_pool_last_window_idx(-1),
+ _curr_tuple(NULL),
+ _dummy_result_tuple(NULL),
+ _curr_partition_idx(-1),
+ _prev_input_row(NULL),
+ _block_mgr_client(nullptr),
+ _input_eos(false),
+ _evaluation_timer(NULL) {
if (tnode.analytic_node.__isset.buffered_tuple_id) {
- _buffered_tuple_desc = descs.get_tuple_descriptor(
- tnode.analytic_node.buffered_tuple_id);
+ _buffered_tuple_desc = descs.get_tuple_descriptor(tnode.analytic_node.buffered_tuple_id);
}
if (!tnode.analytic_node.__isset.window) {
_fn_scope = AnalyticEvalNode::PARTITION;
} else if (tnode.analytic_node.window.type == TAnalyticWindowType::RANGE) {
_fn_scope = AnalyticEvalNode::RANGE;
- DCHECK(!_window.__isset.window_start)
- << "RANGE windows must have UNBOUNDED PRECEDING";
+ DCHECK(!_window.__isset.window_start) << "RANGE windows must have UNBOUNDED PRECEDING";
DCHECK(!_window.__isset.window_end ||
_window.window_end.type == TAnalyticWindowBoundaryType::CURRENT_ROW)
<< "RANGE window end bound must be CURRENT ROW or UNBOUNDED FOLLOWING";
@@ -103,7 +99,7 @@ AnalyticEvalNode::AnalyticEvalNode(ObjectPool* pool, const TPlanNode& tnode,
}
}
- VLOG_ROW << "tnode=" << apache::thrift::ThriftDebugString(tnode);
+ VLOG_ROW << "tnode=" << apache::thrift::ThriftDebugString(tnode);
}
Status AnalyticEvalNode::init(const TPlanNode& tnode, RuntimeState* state) {
@@ -113,8 +109,8 @@ Status AnalyticEvalNode::init(const TPlanNode& tnode, RuntimeState* state) {
for (int i = 0; i < analytic_node.analytic_functions.size(); ++i) {
AggFnEvaluator* evaluator = NULL;
- RETURN_IF_ERROR(AggFnEvaluator::create(
- _pool, analytic_node.analytic_functions[i], true, &evaluator));
+ RETURN_IF_ERROR(AggFnEvaluator::create(_pool, analytic_node.analytic_functions[i], true,
+ &evaluator));
_evaluators.push_back(evaluator);
const TFunction& fn = analytic_node.analytic_functions[i].nodes[0].fn;
_is_lead_fn.push_back("lead" == fn.name.function_name);
@@ -128,14 +124,14 @@ Status AnalyticEvalNode::init(const TPlanNode& tnode, RuntimeState* state) {
if (analytic_node.__isset.partition_by_eq) {
DCHECK(analytic_node.__isset.buffered_tuple_id);
- RETURN_IF_ERROR(Expr::create_expr_tree(
- _pool, analytic_node.partition_by_eq, &_partition_by_eq_expr_ctx));
+ RETURN_IF_ERROR(Expr::create_expr_tree(_pool, analytic_node.partition_by_eq,
+ &_partition_by_eq_expr_ctx));
}
if (analytic_node.__isset.order_by_eq) {
DCHECK(analytic_node.__isset.buffered_tuple_id);
- RETURN_IF_ERROR(Expr::create_expr_tree(
- _pool, analytic_node.order_by_eq, &_order_by_eq_expr_ctx));
+ RETURN_IF_ERROR(
+ Expr::create_expr_tree(_pool, analytic_node.order_by_eq, &_order_by_eq_expr_ctx));
}
return Status::OK();
@@ -182,8 +178,7 @@ Status AnalyticEvalNode::prepare(RuntimeState* state) {
}
}
- _child_tuple_cmp_row = reinterpret_cast<TupleRow*>(
- _mem_pool->allocate(sizeof(Tuple*) * 2));
+ _child_tuple_cmp_row = reinterpret_cast<TupleRow*>(_mem_pool->allocate(sizeof(Tuple*) * 2));
return Status::OK();
}
@@ -193,16 +188,19 @@ Status AnalyticEvalNode::open(RuntimeState* state) {
RETURN_IF_CANCELLED(state);
//RETURN_IF_ERROR(QueryMaintenance(state));
RETURN_IF_ERROR(child(0)->open(state));
- RETURN_IF_ERROR(state->block_mgr2()->register_client(2, mem_tracker(), state, &_block_mgr_client));
- _input_stream.reset(new BufferedTupleStream2(state, child(0)->row_desc(), state->block_mgr2(), _block_mgr_client, false, true));
+ RETURN_IF_ERROR(
+ state->block_mgr2()->register_client(2, mem_tracker(), state, &_block_mgr_client));
+ _input_stream.reset(new BufferedTupleStream2(state, child(0)->row_desc(), state->block_mgr2(),
+ _block_mgr_client, false, true));
RETURN_IF_ERROR(_input_stream->init(id(), runtime_profile(), true));
bool got_read_buffer;
RETURN_IF_ERROR(_input_stream->prepare_for_read(true, &got_read_buffer));
if (!got_read_buffer) {
- std::string msg("Failed to acquire initial read buffer for analytic function "
- "evaluation. Reducing query concurrency or increasing the memory limit may "
- "help this query to complete successfully.");
+ std::string msg(
+ "Failed to acquire initial read buffer for analytic function "
+ "evaluation. Reducing query concurrency or increasing the memory limit may "
+ "help this query to complete successfully.");
return mem_tracker()->MemLimitExceeded(state, msg, -1);
}
@@ -212,10 +210,10 @@ Status AnalyticEvalNode::open(RuntimeState* state) {
RETURN_IF_ERROR(_evaluators[i]->open(state, _fn_ctxs[i]));
if ("first_value_rewrite" == _evaluators[i]->fn_name() &&
- _fn_ctxs[i]->get_num_args() == 2) {
+ _fn_ctxs[i]->get_num_args() == 2) {
DCHECK(!_has_first_val_null_offset);
_first_val_null_offset =
- reinterpret_cast<BigIntVal*>(_fn_ctxs[i]->get_constant_arg(1))->val;
+ reinterpret_cast<BigIntVal*>(_fn_ctxs[i]->get_constant_arg(1))->val;
VLOG_FILE << id() << " FIRST_VAL rewrite null offset: " << _first_val_null_offset;
_has_first_val_null_offset = true;
}
@@ -238,8 +236,10 @@ Status AnalyticEvalNode::open(RuntimeState* state) {
// Fetch the first input batch so that some _prev_input_row can be set here to avoid
// special casing in GetNext().
- _prev_child_batch.reset(new RowBatch(child(0)->row_desc(), state->batch_size(), mem_tracker().get()));
- _curr_child_batch.reset(new RowBatch(child(0)->row_desc(), state->batch_size(), mem_tracker().get()));
+ _prev_child_batch.reset(
+ new RowBatch(child(0)->row_desc(), state->batch_size(), mem_tracker().get()));
+ _curr_child_batch.reset(
+ new RowBatch(child(0)->row_desc(), state->batch_size(), mem_tracker().get()));
while (!_input_eos && _prev_input_row == NULL) {
RETURN_IF_ERROR(child(0)->get_next(state, _curr_child_batch.get(), &_input_eos));
@@ -324,14 +324,13 @@ std::string AnalyticEvalNode::debug_state_string(bool detailed) const {
std::stringstream ss;
ss << "num_returned=" << _input_stream->rows_returned()
<< " num_rows=" << _input_stream->num_rows()
- << " _curr_partition_idx=" << _curr_partition_idx
- << " last_result_idx=" << _last_result_idx;
+ << " _curr_partition_idx=" << _curr_partition_idx << " last_result_idx=" << _last_result_idx;
if (detailed) {
ss << " result_tuples idx: [";
for (std::list<std::pair<int64_t, Tuple*>>::const_iterator it = _result_tuples.begin();
- it != _result_tuples.end(); ++it) {
+ it != _result_tuples.end(); ++it) {
ss << it->first;
if (*it != _result_tuples.back()) {
@@ -345,7 +344,7 @@ std::string AnalyticEvalNode::debug_state_string(bool detailed) const {
ss << " window_tuples idx: [";
for (std::list<std::pair<int64_t, Tuple*>>::const_iterator it = _window_tuples.begin();
- it != _window_tuples.end(); ++it) {
+ it != _window_tuples.end(); ++it) {
ss << it->first;
if (*it != _window_tuples.back()) {
@@ -379,8 +378,7 @@ std::string AnalyticEvalNode::debug_state_string(bool detailed) const {
void AnalyticEvalNode::add_result_tuple(int64_t stream_idx) {
VLOG_ROW << id() << " add_result_tuple idx=" << stream_idx;
DCHECK(_curr_tuple != NULL);
- Tuple* result_tuple = Tuple::create(_result_tuple_desc->byte_size(),
- _curr_tuple_pool.get());
+ Tuple* result_tuple = Tuple::create(_result_tuple_desc->byte_size(), _curr_tuple_pool.get());
AggFnEvaluator::get_value(_evaluators, _fn_ctxs, _curr_tuple, result_tuple);
DCHECK_GT(stream_idx, _last_result_idx);
@@ -390,7 +388,7 @@ void AnalyticEvalNode::add_result_tuple(int64_t stream_idx) {
}
inline void AnalyticEvalNode::try_add_result_tuple_for_prev_row(bool next_partition,
- int64_t stream_idx, TupleRow* row) {
+ int64_t stream_idx, TupleRow* row) {
// The analytic fns are finalized after the previous row if we found a new partition
// or the window is a RANGE and the order by exprs changed. For ROWS windows we do not
// need to compare the current row to the previous row.
@@ -405,8 +403,7 @@ inline void AnalyticEvalNode::try_add_result_tuple_for_prev_row(bool next_partit
}
}
-inline void AnalyticEvalNode::try_add_result_tuple_for_curr_row(int64_t stream_idx,
- TupleRow* row) {
+inline void AnalyticEvalNode::try_add_result_tuple_for_curr_row(int64_t stream_idx, TupleRow* row) {
VLOG_ROW << id() << " try_add_result_tuple_for_curr_row idx=" << stream_idx;
// We only add results at this point for ROWS windows (unless unbounded following)
@@ -446,7 +443,7 @@ inline void AnalyticEvalNode::try_remove_rows_before_window(int64_t stream_idx)
}
inline void AnalyticEvalNode::try_add_remaining_results(int64_t partition_idx,
- int64_t prev_partition_idx) {
+ int64_t prev_partition_idx) {
DCHECK_LT(prev_partition_idx, partition_idx);
// For PARTITION, RANGE, or ROWS with UNBOUNDED PRECEDING: add a result tuple for the
@@ -475,7 +472,7 @@ inline void AnalyticEvalNode::try_add_remaining_results(int64_t partition_idx,
<< " " << debug_state_string(true);
for (int64_t next_result_idx = _last_result_idx + 1; next_result_idx < partition_idx;
- ++next_result_idx) {
+ ++next_result_idx) {
if (_window_tuples.empty()) {
break;
}
@@ -520,32 +517,30 @@ inline void AnalyticEvalNode::init_next_partition(int64_t stream_idx) {
removed_results_past_partition = true;
DCHECK(_window.__isset.window_end &&
_window.window_end.type == TAnalyticWindowBoundaryType::PRECEDING);
- VLOG_ROW << id() << " Removing result past partition idx: "
- << _result_tuples.back().first;
+ VLOG_ROW << id() << " Removing result past partition idx: " << _result_tuples.back().first;
Tuple* prev_result_tuple = _result_tuples.back().second;
_result_tuples.pop_back();
- if (_result_tuples.empty() ||
- _result_tuples.back().first < prev_partition_stream_idx) {
+ if (_result_tuples.empty() || _result_tuples.back().first < prev_partition_stream_idx) {
// prev_result_tuple was the last result tuple in the partition, add it back with
// the index of the last row in the partition so that all output rows in this
// partition get the default result tuple.
_result_tuples.push_back(
- std::pair<int64_t, Tuple*>(_curr_partition_idx - 1, prev_result_tuple));
+ std::pair<int64_t, Tuple*>(_curr_partition_idx - 1, prev_result_tuple));
}
_last_result_idx = _result_tuples.back().first;
}
if (removed_results_past_partition) {
- VLOG_ROW << id() << " After removing results past partition: "
- << debug_state_string(true);
+ VLOG_ROW << id() << " After removing results past partition: " << debug_state_string(true);
DCHECK_EQ(_last_result_idx, _curr_partition_idx - 1);
DCHECK_LE(_input_stream->rows_returned(), _last_result_idx);
}
- if (_fn_scope == ROWS && stream_idx > 0 && (!_window.__isset.window_end ||
- _window.window_end.type == TAnalyticWindowBoundaryType::FOLLOWING)) {
+ if (_fn_scope == ROWS && stream_idx > 0 &&
+ (!_window.__isset.window_end ||
+ _window.window_end.type == TAnalyticWindowBoundaryType::FOLLOWING)) {
try_add_remaining_results(stream_idx, prev_partition_stream_idx);
}
@@ -563,7 +558,7 @@ inline void AnalyticEvalNode::init_next_partition(int64_t stream_idx) {
// count()) for output rows that have no input rows in the window. We need to add this
// result tuple before any input rows are consumed and the evaluators are updated.
if (_fn_scope == ROWS && _window.__isset.window_end &&
- _window.window_end.type == TAnalyticWindowBoundaryType::PRECEDING) {
+ _window.window_end.type == TAnalyticWindowBoundaryType::PRECEDING) {
if (_has_first_val_null_offset) {
// Special handling for FIRST_VALUE which has the window rewritten in the FE
// in order to evaluate the fn efficiently with a trivial agg fn implementation.
@@ -599,8 +594,7 @@ Status AnalyticEvalNode::process_child_batches(RuntimeState* state) {
// Consume child batches until eos or there are enough rows to return more than an
// output batch. Ensuring there is at least one more row left after returning results
// allows us to simplify the logic dealing with _last_result_idx and _result_tuples.
- while (_curr_child_batch.get() != NULL &&
- num_output_rows_ready() < state->batch_size() + 1) {
+ while (_curr_child_batch.get() != NULL && num_output_rows_ready() < state->batch_size() + 1) {
RETURN_IF_CANCELLED(state);
//RETURN_IF_ERROR(QueryMaintenance(state));
RETURN_IF_ERROR(process_child_batch(state));
@@ -666,14 +660,14 @@ Status AnalyticEvalNode::process_child_batch(RuntimeState* state) {
// The _evaluators are updated with the current row.
if (_fn_scope != ROWS || !_window.__isset.window_start ||
- stream_idx - _rows_start_offset >= _curr_partition_idx) {
+ stream_idx - _rows_start_offset >= _curr_partition_idx) {
VLOG_ROW << id() << " Update idx=" << stream_idx;
AggFnEvaluator::add(_evaluators, _fn_ctxs, row, _curr_tuple);
if (_window.__isset.window_start) {
VLOG_ROW << id() << " Adding tuple to window at idx=" << stream_idx;
- Tuple* tuple = row->get_tuple(0)->deep_copy(*_child_tuple_desc,
- _curr_tuple_pool.get());
+ Tuple* tuple =
+ row->get_tuple(0)->deep_copy(*_child_tuple_desc, _curr_tuple_pool.get());
_window_tuples.push_back(std::pair<int64_t, Tuple*>(stream_idx, tuple));
last_window_tuple_idx = stream_idx;
}
@@ -711,26 +705,25 @@ Status AnalyticEvalNode::process_child_batch(RuntimeState* state) {
// Transfer resources to _prev_tuple_pool when enough resources have accumulated
// and the _prev_tuple_pool has already been transferred to an output batch.
- // The memory limit of _curr_tuple_pool is set by the fixed value
+ // The memory limit of _curr_tuple_pool is set by the fixed value
// The size is specified as 8MB, which is used in the extremely strict memory limit.
// Eg: exec_mem_limit < 100MB may cause memory exceeded limit problem. So change it to half of max block size to prevent the problem.
// TODO: Should we keep the buffer of _curr_tuple_pool or release the memory occupied ASAP?
if (_curr_tuple_pool->total_allocated_bytes() > state->block_mgr2()->max_block_size() / 2 &&
- (_prev_pool_last_result_idx == -1 || _prev_pool_last_window_idx == -1)) {
+ (_prev_pool_last_result_idx == -1 || _prev_pool_last_window_idx == -1)) {
_prev_tuple_pool->acquire_data(_curr_tuple_pool.get(), false);
_prev_pool_last_result_idx = _last_result_idx;
_prev_pool_last_window_idx = last_window_tuple_idx;
- VLOG_FILE << id() << " Transfer resources from curr to prev pool at idx: "
- << stream_idx << ", stores tuples with last result idx: "
- << _prev_pool_last_result_idx << " last window idx: "
- << _prev_pool_last_window_idx;
+ VLOG_FILE << id() << " Transfer resources from curr to prev pool at idx: " << stream_idx
+ << ", stores tuples with last result idx: " << _prev_pool_last_result_idx
+ << " last window idx: " << _prev_pool_last_window_idx;
}
return Status::OK();
}
Status AnalyticEvalNode::get_next_output_batch(RuntimeState* state, RowBatch* output_batch,
- bool* eos) {
+ bool* eos) {
SCOPED_TIMER(_evaluation_timer);
VLOG_FILE << id() << " get_next_output_batch: " << debug_state_string(false)
<< " tuple pool size:" << _curr_tuple_pool->total_allocated_bytes();
@@ -837,8 +830,8 @@ Status AnalyticEvalNode::get_next(RuntimeState* state, RowBatch* row_batch, bool
// Transfer resources to the output row batch if enough have accumulated and they're
// no longer needed by output rows to be returned later.
if (_prev_pool_last_result_idx != -1 &&
- _prev_pool_last_result_idx < _input_stream->rows_returned() &&
- _prev_pool_last_window_idx < _window_tuples.front().first) {
+ _prev_pool_last_result_idx < _input_stream->rows_returned() &&
+ _prev_pool_last_window_idx < _window_tuples.front().first) {
VLOG_FILE << id() << " Transfer prev pool to output batch, "
<< " pool size: " << _prev_tuple_pool->total_allocated_bytes()
<< " last result idx: " << _prev_pool_last_result_idx
@@ -935,4 +928,4 @@ void AnalyticEvalNode::debug_string(int indentation_level, std::stringstream* ou
// return ExecNode::QueryMaintenance(state);
//}
-}
+} // namespace doris
diff --git a/be/src/exec/analytic_eval_node.h b/be/src/exec/analytic_eval_node.h
index 8c0f9f4..8c6d7ad 100644
--- a/be/src/exec/analytic_eval_node.h
+++ b/be/src/exec/analytic_eval_node.h
@@ -22,8 +22,8 @@
#include "exprs/expr.h"
//#include "exprs/expr_context.h"
#include "runtime/buffered_block_mgr2.h"
-#include "runtime/buffered_tuple_stream2.inline.h"
#include "runtime/buffered_tuple_stream2.h"
+#include "runtime/buffered_tuple_stream2.inline.h"
#include "runtime/tuple.h"
#include "thrift/protocol/TDebugProtocol.h"
@@ -133,8 +133,7 @@ private:
// add_result_tuple() with the index of the previous row in _input_stream. next_partition
// indicates if the current row is the start of a new partition. stream_idx is the
// index of the current input row from _input_stream.
- void try_add_result_tuple_for_prev_row(bool next_partition, int64_t stream_idx,
- TupleRow* row);
+ void try_add_result_tuple_for_prev_row(bool next_partition, int64_t stream_idx, TupleRow* row);
// Determines if there is a window ending at the current row, and if so, calls
// add_result_tuple() with the index of the current row in _input_stream. stream_idx is
@@ -330,6 +329,6 @@ private:
RuntimeProfile::Counter* _evaluation_timer;
};
-}
+} // namespace doris
#endif
diff --git a/be/src/exec/base_scanner.cpp b/be/src/exec/base_scanner.cpp
index 1ed73e2..8ec57d4 100644
--- a/be/src/exec/base_scanner.cpp
+++ b/be/src/exec/base_scanner.cpp
@@ -15,15 +15,14 @@
// specific language governing permissions and limitations
// under the License.
-
#include "base_scanner.h"
#include "common/logging.h"
#include "runtime/descriptors.h"
#include "runtime/mem_tracker.h"
#include "runtime/raw_value.h"
-#include "runtime/tuple.h"
#include "runtime/runtime_state.h"
+#include "runtime/tuple.h"
namespace doris {
@@ -37,7 +36,8 @@ BaseScanner::BaseScanner(RuntimeState* state, RuntimeProfile* profile,
#if BE_TEST
_mem_tracker(new MemTracker()),
#else
- _mem_tracker(MemTracker::CreateTracker(-1, "Broker Scanner", state->instance_mem_tracker())),
+ _mem_tracker(
+ MemTracker::CreateTracker(-1, "Broker Scanner", state->instance_mem_tracker())),
#endif
_mem_pool(_mem_tracker.get()),
_dest_tuple_desc(nullptr),
@@ -65,7 +65,7 @@ Status BaseScanner::open() {
Status BaseScanner::init_expr_ctxes() {
// Construct _src_slot_descs
const TupleDescriptor* src_tuple_desc =
- _state->desc_tbl().get_tuple_descriptor(_params.src_tuple_id);
+ _state->desc_tbl().get_tuple_descriptor(_params.src_tuple_id);
if (src_tuple_desc == nullptr) {
std::stringstream ss;
ss << "Unknown source tuple descriptor, tuple_id=" << _params.src_tuple_id;
@@ -86,8 +86,8 @@ Status BaseScanner::init_expr_ctxes() {
_src_slot_descs.emplace_back(it->second);
}
// Construct source tuple and tuple row
- _src_tuple = (Tuple*) _mem_pool.allocate(src_tuple_desc->byte_size());
- _src_tuple_row = (TupleRow*) _mem_pool.allocate(sizeof(Tuple*));
+ _src_tuple = (Tuple*)_mem_pool.allocate(src_tuple_desc->byte_size());
+ _src_tuple_row = (TupleRow*)_mem_pool.allocate(sizeof(Tuple*));
_src_tuple_row->set_tuple(0, _src_tuple);
_row_desc.reset(new RowDescriptor(_state->desc_tbl(),
std::vector<TupleId>({_params.src_tuple_id}),
@@ -110,7 +110,7 @@ Status BaseScanner::init_expr_ctxes() {
if (it == std::end(_params.expr_of_dest_slot)) {
std::stringstream ss;
ss << "No expr for dest slot, id=" << slot_desc->id()
- << ", name=" << slot_desc->col_name();
+ << ", name=" << slot_desc->col_name();
return Status::InternalError(ss.str());
}
ExprContext* ctx = nullptr;
@@ -125,9 +125,9 @@ Status BaseScanner::init_expr_ctxes() {
} else {
auto _src_slot_it = src_slot_desc_map.find(it->second);
if (_src_slot_it == std::end(src_slot_desc_map)) {
- std::stringstream ss;
- ss << "No src slot " << it->second << " in src slot descs";
- return Status::InternalError(ss.str());
+ std::stringstream ss;
+ ss << "No src slot " << it->second << " in src slot descs";
+ return Status::InternalError(ss.str());
}
_src_slot_descs_order_by_dest.emplace_back(_src_slot_it->second);
}
@@ -150,7 +150,8 @@ bool BaseScanner::fill_dest_tuple(Tuple* dest_tuple, MemPool* mem_pool) {
// Only when the expr return value is null, we will check the error message.
std::string expr_error = ctx->get_error_msg();
if (!expr_error.empty()) {
- _state->append_error_msg_to_file(_src_tuple_row->to_string(*(_row_desc.get())), expr_error);
+ _state->append_error_msg_to_file(_src_tuple_row->to_string(*(_row_desc.get())),
+ expr_error);
_counter->num_rows_filtered++;
// The ctx is reused, so must clear the error state and message.
ctx->clear_error_msg();
@@ -158,18 +159,21 @@ bool BaseScanner::fill_dest_tuple(Tuple* dest_tuple, MemPool* mem_pool) {
}
// If _strict_mode is false, _src_slot_descs_order_by_dest size could be zero
if (_strict_mode && (_src_slot_descs_order_by_dest[dest_index] != nullptr) &&
- !_src_tuple->is_null(_src_slot_descs_order_by_dest[dest_index]->null_indicator_offset())) {
+ !_src_tuple->is_null(
+ _src_slot_descs_order_by_dest[dest_index]->null_indicator_offset())) {
//Type of the slot is must be Varchar in _src_tuple.
- StringValue* raw_value = _src_tuple->get_string_slot(_src_slot_descs_order_by_dest[dest_index]->tuple_offset());
+ StringValue* raw_value = _src_tuple->get_string_slot(
+ _src_slot_descs_order_by_dest[dest_index]->tuple_offset());
std::string raw_string;
- if (raw_value != nullptr) {//is not null then get raw value
+ if (raw_value != nullptr) { //is not null then get raw value
raw_string = raw_value->to_string();
}
std::stringstream error_msg;
error_msg << "column(" << slot_desc->col_name() << ") value is incorrect "
- << "while strict mode is " << std::boolalpha << _strict_mode
- << ", src value is " << raw_string;
- _state->append_error_msg_to_file(_src_tuple_row->to_string(*(_row_desc.get())), error_msg.str());
+ << "while strict mode is " << std::boolalpha << _strict_mode
+ << ", src value is " << raw_string;
+ _state->append_error_msg_to_file(_src_tuple_row->to_string(*(_row_desc.get())),
+ error_msg.str());
_counter->num_rows_filtered++;
return false;
}
@@ -177,7 +181,8 @@ bool BaseScanner::fill_dest_tuple(Tuple* dest_tuple, MemPool* mem_pool) {
std::stringstream error_msg;
error_msg << "column(" << slot_desc->col_name() << ") value is null "
<< "while columns is not nullable";
- _state->append_error_msg_to_file(_src_tuple_row->to_string(*(_row_desc.get())), error_msg.str());
+ _state->append_error_msg_to_file(_src_tuple_row->to_string(*(_row_desc.get())),
+ error_msg.str());
_counter->num_rows_filtered++;
return false;
}
@@ -194,7 +199,8 @@ bool BaseScanner::fill_dest_tuple(Tuple* dest_tuple, MemPool* mem_pool) {
return true;
}
-void BaseScanner::fill_slots_of_columns_from_path(int start, const std::vector<std::string>& columns_from_path) {
+void BaseScanner::fill_slots_of_columns_from_path(
+ int start, const std::vector<std::string>& columns_from_path) {
// values of columns from path can not be null
for (int i = 0; i < columns_from_path.size(); ++i) {
auto slot_desc = _src_slot_descs.at(i + start);
@@ -207,4 +213,4 @@ void BaseScanner::fill_slots_of_columns_from_path(int start, const std::vector<s
}
}
-}
+} // namespace doris
diff --git a/be/src/exec/base_scanner.h b/be/src/exec/base_scanner.h
index 6be496f..1fc2949 100644
--- a/be/src/exec/base_scanner.h
+++ b/be/src/exec/base_scanner.h
@@ -19,8 +19,8 @@
#define BE_SRC_EXEC_BASE_SCANNER_H_
#include "common/status.h"
-#include "runtime/tuple.h"
#include "exprs/expr.h"
+#include "runtime/tuple.h"
#include "util/runtime_profile.h"
namespace doris {
@@ -34,21 +34,17 @@ class RuntimeState;
class ExprContext;
struct ScannerCounter {
- ScannerCounter() :
- num_rows_filtered(0),
- num_rows_unselected(0) {
- }
+ ScannerCounter() : num_rows_filtered(0), num_rows_unselected(0) {}
- int64_t num_rows_filtered; // unqualified rows (unmatched the dest schema, or no partition)
+ int64_t num_rows_filtered; // unqualified rows (unmatched the dest schema, or no partition)
int64_t num_rows_unselected; // rows filtered by predicates
};
class BaseScanner {
public:
- BaseScanner(RuntimeState* state, RuntimeProfile* profile, const TBrokerScanRangeParams& params, ScannerCounter* counter);
- virtual ~BaseScanner() {
- Expr::close(_dest_expr_ctx, _state);
- };
+ BaseScanner(RuntimeState* state, RuntimeProfile* profile, const TBrokerScanRangeParams& params,
+ ScannerCounter* counter);
+ virtual ~BaseScanner() { Expr::close(_dest_expr_ctx, _state); };
virtual Status init_expr_ctxes();
// Open this scanner, will initialize information need to
@@ -61,7 +57,8 @@ public:
virtual void close() = 0;
bool fill_dest_tuple(Tuple* dest_tuple, MemPool* mem_pool);
- void fill_slots_of_columns_from_path(int start, const std::vector<std::string>& columns_from_path);
+ void fill_slots_of_columns_from_path(int start,
+ const std::vector<std::string>& columns_from_path);
protected:
RuntimeState* _state;
diff --git a/be/src/exec/blocking_join_node.cpp b/be/src/exec/blocking_join_node.cpp
index 7f60b81..b4b2f05 100644
--- a/be/src/exec/blocking_join_node.cpp
+++ b/be/src/exec/blocking_join_node.cpp
@@ -20,22 +20,17 @@
#include <sstream>
#include "exprs/expr.h"
+#include "gen_cpp/PlanNodes_types.h"
#include "runtime/row_batch.h"
#include "runtime/runtime_state.h"
#include "util/runtime_profile.h"
-#include "gen_cpp/PlanNodes_types.h"
namespace doris {
-BlockingJoinNode::BlockingJoinNode(const std::string& node_name,
- const TJoinOp::type join_op,
- ObjectPool* pool,
- const TPlanNode& tnode,
+BlockingJoinNode::BlockingJoinNode(const std::string& node_name, const TJoinOp::type join_op,
+ ObjectPool* pool, const TPlanNode& tnode,
const DescriptorTbl& descs)
- : ExecNode(pool, tnode, descs),
- _node_name(node_name),
- _join_op(join_op) {
-}
+ : ExecNode(pool, tnode, descs), _node_name(node_name), _join_op(join_op) {}
Status BlockingJoinNode::init(const TPlanNode& tnode, RuntimeState* state) {
return ExecNode::init(tnode, state);
@@ -54,8 +49,7 @@ Status BlockingJoinNode::prepare(RuntimeState* state) {
_build_timer = ADD_TIMER(runtime_profile(), "BuildTime");
_left_child_timer = ADD_TIMER(runtime_profile(), "LeftChildTime");
_build_row_counter = ADD_COUNTER(runtime_profile(), "BuildRows", TUnit::UNIT);
- _left_child_row_counter = ADD_COUNTER(runtime_profile(), "LeftChildRows",
- TUnit::UNIT);
+ _left_child_row_counter = ADD_COUNTER(runtime_profile(), "LeftChildRows", TUnit::UNIT);
_result_tuple_row_size = _row_descriptor.tuple_descriptors().size() * sizeof(Tuple*);
@@ -167,8 +161,7 @@ Status BlockingJoinNode::open(RuntimeState* state) {
void BlockingJoinNode::debug_string(int indentation_level, std::stringstream* out) const {
*out << std::string(indentation_level * 2, ' ');
*out << _node_name;
- *out << "(eos=" << (_eos ? "true" : "false")
- << " left_batch_pos=" << _left_batch_pos;
+ *out << "(eos=" << (_eos ? "true" : "false") << " left_batch_pos=" << _left_batch_pos;
add_to_debug_string(indentation_level, out);
ExecNode::debug_string(indentation_level, out);
*out << ")";
@@ -185,7 +178,7 @@ std::string BlockingJoinNode::get_left_child_row_string(TupleRow* row) {
}
int* is_build_tuple =
- std::find(_build_tuple_idx_ptr, _build_tuple_idx_ptr + _build_tuple_size, i);
+ std::find(_build_tuple_idx_ptr, _build_tuple_idx_ptr + _build_tuple_size, i);
if (is_build_tuple != _build_tuple_idx_ptr + _build_tuple_size) {
out << Tuple::to_string(NULL, *row_desc().tuple_descriptors()[i]);
@@ -214,4 +207,4 @@ void BlockingJoinNode::create_output_row(TupleRow* out, TupleRow* left, TupleRow
}
}
-}
+} // namespace doris
diff --git a/be/src/exec/blocking_join_node.h b/be/src/exec/blocking_join_node.h
index e35dde6..f0406d3 100644
--- a/be/src/exec/blocking_join_node.h
+++ b/be/src/exec/blocking_join_node.h
@@ -35,8 +35,8 @@ class TupleRow;
// right child in open().
class BlockingJoinNode : public ExecNode {
public:
- BlockingJoinNode(const std::string& node_name, const TJoinOp::type join_op,
- ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs);
+ BlockingJoinNode(const std::string& node_name, const TJoinOp::type join_op, ObjectPool* pool,
+ const TPlanNode& tnode, const DescriptorTbl& descs);
virtual ~BlockingJoinNode();
@@ -60,15 +60,15 @@ public:
private:
const std::string _node_name;
TJoinOp::type _join_op;
- bool _eos; // if true, nothing left to return in get_next()
- boost::scoped_ptr<MemPool> _build_pool; // holds everything referenced from build side
+ bool _eos; // if true, nothing left to return in get_next()
+ boost::scoped_ptr<MemPool> _build_pool; // holds everything referenced from build side
// _left_batch must be cleared before calling get_next(). The child node
// does not initialize all tuple ptrs in the row, only the ones that it
// is responsible for.
boost::scoped_ptr<RowBatch> _left_batch;
- int _left_batch_pos; // current scan pos in _left_batch
- bool _left_side_eos; // if true, left child has no more rows to process
+ int _left_batch_pos; // current scan pos in _left_batch
+ bool _left_side_eos; // if true, left child has no more rows to process
TupleRow* _current_left_child_row;
// _build_tuple_idx[i] is the tuple index of child(1)'s tuple[i] in the output row
@@ -84,10 +84,10 @@ private:
// This should be the same size as the left child tuple row.
int _result_tuple_row_size;
- RuntimeProfile::Counter* _build_timer; // time to prepare build side
- RuntimeProfile::Counter* _left_child_timer; // time to process left child batch
- RuntimeProfile::Counter* _build_row_counter; // num build rows
- RuntimeProfile::Counter* _left_child_row_counter; // num left child rows
+ RuntimeProfile::Counter* _build_timer; // time to prepare build side
+ RuntimeProfile::Counter* _left_child_timer; // time to process left child batch
+ RuntimeProfile::Counter* _build_row_counter; // num build rows
+ RuntimeProfile::Counter* _left_child_row_counter; // num left child rows
// Init the build-side state for a new left child row (e.g. hash table iterator or list
// iterator) given the first row. Used in open() to prepare for get_next().
@@ -102,8 +102,7 @@ private:
// Gives subclasses an opportunity to add debug output to the debug string printed by
// debug_string().
- virtual void add_to_debug_string(int indentation_level, std::stringstream* out) const {
- }
+ virtual void add_to_debug_string(int indentation_level, std::stringstream* out) const {}
// Subclasses should not override, use add_to_debug_string() to add to the result.
virtual void debug_string(int indentation_level, std::stringstream* out) const;
@@ -122,12 +121,13 @@ private:
void create_output_row(TupleRow* out_row, TupleRow* left_row, TupleRow* build_row);
friend class CrossJoinNode;
+
private:
// Supervises ConstructBuildSide in a separate thread, and returns its status in the
// promise parameter.
void build_side_thread(RuntimeState* state, boost::promise<Status>* status);
};
-}
+} // namespace doris
#endif
diff --git a/be/src/exec/broker_reader.cpp b/be/src/exec/broker_reader.cpp
index 2125bb3..e5f67b6 100644
--- a/be/src/exec/broker_reader.cpp
+++ b/be/src/exec/broker_reader.cpp
@@ -155,7 +155,8 @@ Status BrokerReader::readat(int64_t position, int64_t nbytes, int64_t* bytes_rea
return status;
}
- VLOG_RPC << "send pread request to broker:" << broker_addr << " position:" << position << ", read bytes length:" << nbytes;
+ VLOG_RPC << "send pread request to broker:" << broker_addr << " position:" << position
+ << ", read bytes length:" << nbytes;
try {
client->pread(response, request);
@@ -255,4 +256,3 @@ void BrokerReader::close() {
}
} // namespace doris
-
diff --git a/be/src/exec/broker_reader.h b/be/src/exec/broker_reader.h
index 6aa5c06..4730260 100644
--- a/be/src/exec/broker_reader.h
+++ b/be/src/exec/broker_reader.h
@@ -19,13 +19,13 @@
#include <stdint.h>
-#include <string>
#include <map>
+#include <string>
#include "common/status.h"
#include "exec/file_reader.h"
-#include "gen_cpp/Types_types.h"
#include "gen_cpp/PaloBrokerService_types.h"
+#include "gen_cpp/Types_types.h"
namespace doris {
@@ -39,19 +39,17 @@ class BrokerReader : public FileReader {
public:
// If the reader need the file size, set it when construct BrokerReader.
// There is no other way to set the file size.
- BrokerReader(ExecEnv* env,
- const std::vector<TNetworkAddress>& broker_addresses,
- const std::map<std::string, std::string>& properties,
- const std::string& path,
- int64_t start_offset,
- int64_t file_size = 0);
+ BrokerReader(ExecEnv* env, const std::vector<TNetworkAddress>& broker_addresses,
+ const std::map<std::string, std::string>& properties, const std::string& path,
+ int64_t start_offset, int64_t file_size = 0);
virtual ~BrokerReader();
virtual Status open() override;
- // Read
+ // Read
virtual Status read(uint8_t* buf, size_t* buf_len, bool* eof) override;
- virtual Status readat(int64_t position, int64_t nbytes, int64_t* bytes_read, void* out) override;
+ virtual Status readat(int64_t position, int64_t nbytes, int64_t* bytes_read,
+ void* out) override;
virtual Status read_one_message(uint8_t** buf, size_t* length) override;
virtual int64_t size() override;
virtual Status seek(int64_t position) override;
@@ -74,5 +72,4 @@ private:
int _addr_idx;
};
-}
-
+} // namespace doris
diff --git a/be/src/exec/broker_scan_node.cpp b/be/src/exec/broker_scan_node.cpp
index 1acf4a1..233c6c6 100644
--- a/be/src/exec/broker_scan_node.cpp
+++ b/be/src/exec/broker_scan_node.cpp
@@ -21,32 +21,29 @@
#include <sstream>
#include "common/object_pool.h"
-#include "runtime/runtime_state.h"
-#include "runtime/row_batch.h"
-#include "runtime/dpp_sink_internal.h"
#include "exec/broker_scanner.h"
-#include "exec/parquet_scanner.h"
-#include "exec/orc_scanner.h"
#include "exec/json_scanner.h"
+#include "exec/orc_scanner.h"
+#include "exec/parquet_scanner.h"
#include "exprs/expr.h"
+#include "runtime/dpp_sink_internal.h"
+#include "runtime/row_batch.h"
+#include "runtime/runtime_state.h"
#include "util/runtime_profile.h"
namespace doris {
-BrokerScanNode::BrokerScanNode(
- ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs) :
- ScanNode(pool, tnode, descs),
- _tuple_id(tnode.broker_scan_node.tuple_id),
- _runtime_state(nullptr),
- _tuple_desc(nullptr),
- _num_running_scanners(0),
- _scan_finished(false),
- _max_buffered_batches(32),
- _wait_scanner_timer(nullptr) {
-}
+BrokerScanNode::BrokerScanNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs)
+ : ScanNode(pool, tnode, descs),
+ _tuple_id(tnode.broker_scan_node.tuple_id),
+ _runtime_state(nullptr),
+ _tuple_desc(nullptr),
+ _num_running_scanners(0),
+ _scan_finished(false),
+ _max_buffered_batches(32),
+ _wait_scanner_timer(nullptr) {}
-BrokerScanNode::~BrokerScanNode() {
-}
+BrokerScanNode::~BrokerScanNode() {}
// We use the PartitionRange to compare here. It should not be a member function of PartitionInfo
// class because there are some other member in it.
@@ -59,17 +56,15 @@ Status BrokerScanNode::init(const TPlanNode& tnode, RuntimeState* state) {
auto& broker_scan_node = tnode.broker_scan_node;
if (broker_scan_node.__isset.partition_exprs) {
// ASSERT broker_scan_node.__isset.partition_infos == true
- RETURN_IF_ERROR(Expr::create_expr_trees(
- _pool, broker_scan_node.partition_exprs, &_partition_expr_ctxs));
+ RETURN_IF_ERROR(Expr::create_expr_trees(_pool, broker_scan_node.partition_exprs,
+ &_partition_expr_ctxs));
for (auto& t_partition_info : broker_scan_node.partition_infos) {
PartitionInfo* info = _pool->add(new PartitionInfo());
RETURN_IF_ERROR(PartitionInfo::from_thrift(_pool, t_partition_info, info));
_partition_infos.emplace_back(info);
}
// partitions should be in ascending order
- std::sort(_partition_infos.begin(),
- _partition_infos.end(),
- compare_part_use_range);
+ std::sort(_partition_infos.begin(), _partition_infos.end(), compare_part_use_range);
}
return Status::OK();
}
@@ -98,8 +93,7 @@ Status BrokerScanNode::prepare(RuntimeState* state) {
// prepare partition
if (_partition_expr_ctxs.size() > 0) {
- RETURN_IF_ERROR(Expr::prepare(
- _partition_expr_ctxs, state, row_desc(), expr_mem_tracker()));
+ RETURN_IF_ERROR(Expr::prepare(_partition_expr_ctxs, state, row_desc(), expr_mem_tracker()));
for (auto iter : _partition_infos) {
RETURN_IF_ERROR(iter->prepare(state, row_desc(), expr_mem_tracker()));
}
@@ -158,10 +152,8 @@ Status BrokerScanNode::get_next(RuntimeState* state, RowBatch* row_batch, bool*
std::shared_ptr<RowBatch> scanner_batch;
{
std::unique_lock<std::mutex> l(_batch_queue_lock);
- while (_process_status.ok() &&
- !_runtime_state->is_cancelled() &&
- _num_running_scanners > 0 &&
- _batch_queue.empty()) {
+ while (_process_status.ok() && !_runtime_state->is_cancelled() &&
+ _num_running_scanners > 0 && _batch_queue.empty()) {
SCOPED_TIMER(_wait_scanner_timer);
_queue_reader_cond.wait_for(l, std::chrono::seconds(1));
}
@@ -215,10 +207,10 @@ Status BrokerScanNode::get_next(RuntimeState* state, RowBatch* row_batch, bool*
for (int i = 0; i < row_batch->num_rows(); ++i) {
TupleRow* row = row_batch->get_row(i);
VLOG_ROW << "BrokerScanNode output row: "
- << Tuple::to_string(row->get_tuple(0), *_tuple_desc);
+ << Tuple::to_string(row->get_tuple(0), *_tuple_desc);
}
}
-
+
return Status::OK();
}
@@ -243,7 +235,7 @@ Status BrokerScanNode::close(RuntimeState* state) {
}
}
- // Close
+ // Close
_batch_queue.clear();
return ExecNode::close(state);
@@ -254,7 +246,7 @@ Status BrokerScanNode::set_scan_ranges(const std::vector<TScanRangeParams>& scan
_scan_ranges = scan_ranges;
// Now we initialize partition information
- if (_partition_expr_ctxs.size() > 0) {
+ if (_partition_expr_ctxs.size() > 0) {
for (auto& range : _scan_ranges) {
auto& params = range.scan_range.broker_scan_range.params;
if (params.__isset.partition_ids) {
@@ -271,61 +263,42 @@ void BrokerScanNode::debug_string(int ident_level, std::stringstream* out) const
}
std::unique_ptr<BaseScanner> BrokerScanNode::create_scanner(const TBrokerScanRange& scan_range,
- ScannerCounter* counter) {
- BaseScanner *scan = nullptr;
+ ScannerCounter* counter) {
+ BaseScanner* scan = nullptr;
switch (scan_range.ranges[0].format_type) {
case TFileFormatType::FORMAT_PARQUET:
- scan = new ParquetScanner(_runtime_state,
- runtime_profile(),
- scan_range.params,
- scan_range.ranges,
- scan_range.broker_addresses,
- counter);
+ scan = new ParquetScanner(_runtime_state, runtime_profile(), scan_range.params,
+ scan_range.ranges, scan_range.broker_addresses, counter);
break;
case TFileFormatType::FORMAT_ORC:
- scan = new ORCScanner(_runtime_state,
- runtime_profile(),
- scan_range.params,
- scan_range.ranges,
- scan_range.broker_addresses,
- counter);
+ scan = new ORCScanner(_runtime_state, runtime_profile(), scan_range.params,
+ scan_range.ranges, scan_range.broker_addresses, counter);
break;
case TFileFormatType::FORMAT_JSON:
- scan = new JsonScanner(_runtime_state,
- runtime_profile(),
- scan_range.params,
- scan_range.ranges,
- scan_range.broker_addresses,
- counter);
+ scan = new JsonScanner(_runtime_state, runtime_profile(), scan_range.params,
+ scan_range.ranges, scan_range.broker_addresses, counter);
break;
default:
- scan = new BrokerScanner(
- _runtime_state,
- runtime_profile(),
- scan_range.params,
- scan_range.ranges,
- scan_range.broker_addresses,
- counter);
+ scan = new BrokerScanner(_runtime_state, runtime_profile(), scan_range.params,
+ scan_range.ranges, scan_range.broker_addresses, counter);
}
std::unique_ptr<BaseScanner> scanner(scan);
return scanner;
}
-Status BrokerScanNode::scanner_scan(
- const TBrokerScanRange& scan_range,
- const std::vector<ExprContext*>& conjunct_ctxs,
- const std::vector<ExprContext*>& partition_expr_ctxs,
- ScannerCounter* counter) {
-
+Status BrokerScanNode::scanner_scan(const TBrokerScanRange& scan_range,
+ const std::vector<ExprContext*>& conjunct_ctxs,
+ const std::vector<ExprContext*>& partition_expr_ctxs,
+ ScannerCounter* counter) {
//create scanner object and open
std::unique_ptr<BaseScanner> scanner = create_scanner(scan_range, counter);
RETURN_IF_ERROR(scanner->open());
bool scanner_eof = false;
-
+
while (!scanner_eof) {
// Fill one row batch
std::shared_ptr<RowBatch> row_batch(
- new RowBatch(row_desc(), _runtime_state->batch_size(), mem_tracker().get()));
+ new RowBatch(row_desc(), _runtime_state->batch_size(), mem_tracker().get()));
// create new tuple buffer for row_batch
MemPool* tuple_pool = row_batch->tuple_data_pool();
@@ -375,14 +348,13 @@ Status BrokerScanNode::scanner_scan(
// Row batch has been filled, push this to the queue
if (row_batch->num_rows() > 0) {
std::unique_lock<std::mutex> l(_batch_queue_lock);
- while (_process_status.ok() &&
- !_scan_finished.load() &&
+ while (_process_status.ok() && !_scan_finished.load() &&
!_runtime_state->is_cancelled() &&
- // stop pushing more batch if
- // 1. too many batches in queue, or
- // 2. at least one batch in queue and memory exceed limit.
- (_batch_queue.size() >= _max_buffered_batches
- || (mem_tracker()->AnyLimitExceeded(MemLimit::HARD) && !_batch_queue.empty()))) {
+ // stop pushing more batch if
+ // 1. too many batches in queue, or
+ // 2. at least one batch in queue and memory exceed limit.
+ (_batch_queue.size() >= _max_buffered_batches ||
+ (mem_tracker()->AnyLimitExceeded(MemLimit::HARD) && !_batch_queue.empty()))) {
_queue_writer_cond.wait_for(l, std::chrono::seconds(1));
}
// Process already set failed, so we just return OK
@@ -400,7 +372,7 @@ Status BrokerScanNode::scanner_scan(
// Queue size Must be smaller than _max_buffered_batches
_batch_queue.push_back(row_batch);
- // Notify reader to
+ // Notify reader to
_queue_reader_cond.notify_one();
}
}
@@ -415,22 +387,23 @@ void BrokerScanNode::scanner_worker(int start_idx, int length) {
if (!status.ok()) {
LOG(WARNING) << "Clone conjuncts failed.";
}
- std::vector<ExprContext*> partition_expr_ctxs;;
+ std::vector<ExprContext*> partition_expr_ctxs;
+ ;
if (status.ok()) {
- status = Expr::clone_if_not_exists(
- _partition_expr_ctxs, _runtime_state, &partition_expr_ctxs);
+ status = Expr::clone_if_not_exists(_partition_expr_ctxs, _runtime_state,
+ &partition_expr_ctxs);
if (!status.ok()) {
LOG(WARNING) << "Clone conjuncts failed.";
}
}
ScannerCounter counter;
for (int i = 0; i < length && status.ok(); ++i) {
- const TBrokerScanRange& scan_range =
- _scan_ranges[start_idx + i].scan_range.broker_scan_range;
+ const TBrokerScanRange& scan_range =
+ _scan_ranges[start_idx + i].scan_range.broker_scan_range;
status = scanner_scan(scan_range, scanner_expr_ctxs, partition_expr_ctxs, &counter);
if (!status.ok()) {
- LOG(WARNING) << "Scanner[" << start_idx + i << "] process failed. status="
- << status.get_error_msg();
+ LOG(WARNING) << "Scanner[" << start_idx + i
+ << "] process failed. status=" << status.get_error_msg();
}
}
@@ -438,7 +411,7 @@ void BrokerScanNode::scanner_worker(int start_idx, int length) {
_runtime_state->update_num_rows_load_filtered(counter.num_rows_filtered);
_runtime_state->update_num_rows_load_unselected(counter.num_rows_unselected);
- // scanner is going to finish
+ // scanner is going to finish
{
std::lock_guard<std::mutex> l(_batch_queue_lock);
if (!status.ok()) {
@@ -475,8 +448,8 @@ int64_t BrokerScanNode::binary_find_partition_id(const PartRangeKey& key) const
return -1;
}
-int64_t BrokerScanNode::get_partition_id(
- const std::vector<ExprContext*>& partition_expr_ctxs, TupleRow* row) const {
+int64_t BrokerScanNode::get_partition_id(const std::vector<ExprContext*>& partition_expr_ctxs,
+ TupleRow* row) const {
if (_partition_infos.size() == 0) {
return -1;
}
@@ -494,4 +467,4 @@ int64_t BrokerScanNode::get_partition_id(
return binary_find_partition_id(part_key);
}
-}
+} // namespace doris
diff --git a/be/src/exec/broker_scan_node.h b/be/src/exec/broker_scan_node.h
index 9f37d31..dedd358 100644
--- a/be/src/exec/broker_scan_node.h
+++ b/be/src/exec/broker_scan_node.h
@@ -20,10 +20,10 @@
#include <atomic>
#include <condition_variable>
#include <map>
-#include <string>
-#include <vector>
#include <mutex>
+#include <string>
#include <thread>
+#include <vector>
#include "base_scanner.h"
#include "common/status.h"
@@ -64,8 +64,7 @@ public:
// If there is no partition information, return -1
// Return partition id if we find the partition match this row,
// return -1, if there is no such partition.
- int64_t get_partition_id(
- const std::vector<ExprContext*>& partition_exprs, TupleRow* row) const;
+ int64_t get_partition_id(const std::vector<ExprContext*>& partition_exprs, TupleRow* row) const;
protected:
// Write debug string of this into out.
@@ -98,7 +97,7 @@ private:
int64_t binary_find_partition_id(const PartRangeKey& key) const;
std::unique_ptr<BaseScanner> create_scanner(const TBrokerScanRange& scan_range,
- ScannerCounter* counter);
+ ScannerCounter* counter);
private:
TupleId _tuple_id;
@@ -133,4 +132,4 @@ private:
RuntimeProfile::Counter* _wait_scanner_timer;
};
-}
+} // namespace doris
diff --git a/be/src/exec/broker_scanner.cpp b/be/src/exec/broker_scanner.cpp
index 39bbaa2..4741163 100644
--- a/be/src/exec/broker_scanner.cpp
+++ b/be/src/exec/broker_scanner.cpp
@@ -17,9 +17,16 @@
#include "exec/broker_scanner.h"
-#include <sstream>
#include <iostream>
+#include <sstream>
+#include "exec/broker_reader.h"
+#include "exec/decompressor.h"
+#include "exec/local_file_reader.h"
+#include "exec/plain_text_line_reader.h"
+#include "exec/text_converter.h"
+#include "exec/text_converter.hpp"
+#include "exprs/expr.h"
#include "runtime/descriptors.h"
#include "runtime/exec_env.h"
#include "runtime/mem_tracker.h"
@@ -27,44 +34,36 @@
#include "runtime/stream_load/load_stream_mgr.h"
#include "runtime/stream_load/stream_load_pipe.h"
#include "runtime/tuple.h"
-#include "exprs/expr.h"
-#include "exec/text_converter.h"
-#include "exec/text_converter.hpp"
-#include "exec/plain_text_line_reader.h"
-#include "exec/local_file_reader.h"
-#include "exec/broker_reader.h"
-#include "exec/decompressor.h"
#include "util/utf8_check.h"
namespace doris {
-BrokerScanner::BrokerScanner(RuntimeState* state,
- RuntimeProfile* profile,
+BrokerScanner::BrokerScanner(RuntimeState* state, RuntimeProfile* profile,
const TBrokerScanRangeParams& params,
const std::vector<TBrokerRangeDesc>& ranges,
const std::vector<TNetworkAddress>& broker_addresses,
- ScannerCounter* counter) : BaseScanner(state, profile, params, counter),
- _ranges(ranges),
- _broker_addresses(broker_addresses),
- // _splittable(params.splittable),
- _value_separator(static_cast<char>(params.column_separator)),
- _line_delimiter(static_cast<char>(params.line_delimiter)),
- _cur_file_reader(nullptr),
- _cur_line_reader(nullptr),
- _cur_decompressor(nullptr),
- _next_range(0),
- _cur_line_reader_eof(false),
- _scanner_eof(false),
- _skip_next_line(false) {
-}
+ ScannerCounter* counter)
+ : BaseScanner(state, profile, params, counter),
+ _ranges(ranges),
+ _broker_addresses(broker_addresses),
+ // _splittable(params.splittable),
+ _value_separator(static_cast<char>(params.column_separator)),
+ _line_delimiter(static_cast<char>(params.line_delimiter)),
+ _cur_file_reader(nullptr),
+ _cur_line_reader(nullptr),
+ _cur_decompressor(nullptr),
+ _next_range(0),
+ _cur_line_reader_eof(false),
+ _scanner_eof(false),
+ _skip_next_line(false) {}
BrokerScanner::~BrokerScanner() {
close();
}
Status BrokerScanner::open() {
- RETURN_IF_ERROR(BaseScanner::open());// base default function
- _text_converter.reset(new(std::nothrow) TextConverter('\\'));
+ RETURN_IF_ERROR(BaseScanner::open()); // base default function
+ _text_converter.reset(new (std::nothrow) TextConverter('\\'));
if (_text_converter == nullptr) {
return Status::InternalError("No memory error.");
}
@@ -84,8 +83,7 @@ Status BrokerScanner::get_next(Tuple* tuple, MemPool* tuple_pool, bool* eof) {
}
const uint8_t* ptr = nullptr;
size_t size = 0;
- RETURN_IF_ERROR(_cur_line_reader->read_line(
- &ptr, &size, &_cur_line_reader_eof));
+ RETURN_IF_ERROR(_cur_line_reader->read_line(&ptr, &size, &_cur_line_reader_eof));
if (_skip_next_line) {
_skip_next_line = false;
continue;
@@ -147,8 +145,9 @@ Status BrokerScanner::open_file_reader() {
break;
}
case TFileType::FILE_BROKER: {
- BrokerReader* broker_reader = new BrokerReader(
- _state->exec_env(), _broker_addresses, _params.properties, range.path, start_offset);
+ BrokerReader* broker_reader =
+ new BrokerReader(_state->exec_env(), _broker_addresses, _params.properties,
+ range.path, start_offset);
RETURN_IF_ERROR(broker_reader->open());
_cur_file_reader = broker_reader;
break;
@@ -204,8 +203,7 @@ Status BrokerScanner::create_decompressor(TFileFormatType::type type) {
return Status::InternalError(ss.str());
}
}
- RETURN_IF_ERROR(Decompressor::create_decompressor(
- compress_type, &_cur_decompressor));
+ RETURN_IF_ERROR(Decompressor::create_decompressor(compress_type, &_cur_decompressor));
return Status::OK();
}
@@ -247,10 +245,8 @@ Status BrokerScanner::open_line_reader() {
case TFileFormatType::FORMAT_CSV_LZ4FRAME:
case TFileFormatType::FORMAT_CSV_LZOP:
case TFileFormatType::FORMAT_CSV_DEFLATE:
- _cur_line_reader = new PlainTextLineReader(
- _profile,
- _cur_file_reader, _cur_decompressor,
- size, _line_delimiter);
+ _cur_line_reader = new PlainTextLineReader(_profile, _cur_file_reader, _cur_decompressor,
+ size, _line_delimiter);
break;
default: {
std::stringstream ss;
@@ -286,8 +282,7 @@ void BrokerScanner::close() {
}
}
-void BrokerScanner::split_line(
- const Slice& line, std::vector<Slice>* values) {
+void BrokerScanner::split_line(const Slice& line, std::vector<Slice>* values) {
// line-begin char and line-end char are considered to be 'delimiter'
const char* value = line.data;
const char* ptr = line.data;
@@ -300,9 +295,8 @@ void BrokerScanner::split_line(
values->emplace_back(value, ptr - value);
}
-void BrokerScanner::fill_fix_length_string(
- const Slice& value, MemPool* pool,
- char** new_value_p, const int new_value_length) {
+void BrokerScanner::fill_fix_length_string(const Slice& value, MemPool* pool, char** new_value_p,
+ const int new_value_length) {
if (new_value_length != 0 && value.size < new_value_length) {
*new_value_p = reinterpret_cast<char*>(pool->allocate(new_value_length));
@@ -318,18 +312,16 @@ void BrokerScanner::fill_fix_length_string(
// .123 1.23 123. -1.23
// ATTN: The decimal point and (for negative numbers) the "-" sign are not counted.
// like '.123', it will be regarded as '0.123', but it match decimal(3, 3)
-bool BrokerScanner::check_decimal_input(
- const Slice& slice,
- int precision, int scale,
- std::stringstream* error_msg) {
+bool BrokerScanner::check_decimal_input(const Slice& slice, int precision, int scale,
+ std::stringstream* error_msg) {
const char* value = slice.data;
size_t value_length = slice.size;
if (value_length > (precision + 2)) {
(*error_msg) << "the length of decimal value is overflow. "
- << "precision in schema: (" << precision << ", " << scale << "); "
- << "value: [" << slice.to_string() << "]; "
- << "str actual length: " << value_length << ";";
+ << "precision in schema: (" << precision << ", " << scale << "); "
+ << "value: [" << slice.to_string() << "]; "
+ << "str actual length: " << value_length << ";";
return false;
}
@@ -357,7 +349,7 @@ bool BrokerScanner::check_decimal_input(
int value_int_len = 0;
int value_frac_len = 0;
value_int_len = point_index - begin_index;
- value_frac_len = end_index- point_index;
+ value_frac_len = end_index - point_index;
if (point_index == -1) {
// an int value: like 123
@@ -365,33 +357,27 @@ bool BrokerScanner::check_decimal_input(
value_frac_len = 0;
} else {
value_int_len = point_index - begin_index;
- value_frac_len = end_index- point_index;
+ value_frac_len = end_index - point_index;
}
if (value_int_len > (precision - scale)) {
- (*error_msg) << "the int part length longer than schema precision ["
- << precision << "]. "
- << "value [" << slice.to_string() << "]. ";
+ (*error_msg) << "the int part length longer than schema precision [" << precision << "]. "
+ << "value [" << slice.to_string() << "]. ";
return false;
} else if (value_frac_len > scale) {
- (*error_msg) << "the frac part length longer than schema scale ["
- << scale << "]. "
- << "value [" << slice.to_string() << "]. ";
+ (*error_msg) << "the frac part length longer than schema scale [" << scale << "]. "
+ << "value [" << slice.to_string() << "]. ";
return false;
}
return true;
}
bool is_null(const Slice& slice) {
- return slice.size == 2 &&
- slice.data[0] == '\\' &&
- slice.data[1] == 'N';
+ return slice.size == 2 && slice.data[0] == '\\' && slice.data[1] == 'N';
}
// Convert one row to this tuple
-bool BrokerScanner::convert_one_row(
- const Slice& line,
- Tuple* tuple, MemPool* tuple_pool) {
+bool BrokerScanner::convert_one_row(const Slice& line, Tuple* tuple, MemPool* tuple_pool) {
if (!line_to_src_tuple(line)) {
return false;
}
@@ -400,20 +386,16 @@ bool BrokerScanner::convert_one_row(
// Convert one row to this tuple
bool BrokerScanner::line_to_src_tuple(const Slice& line) {
-
if (!validate_utf8(line.data, line.size)) {
std::stringstream error_msg;
error_msg << "data is not encoded by UTF-8";
- _state->append_error_msg_to_file(std::string(line.data, line.size),
- error_msg.str());
+ _state->append_error_msg_to_file(std::string(line.data, line.size), error_msg.str());
_counter->num_rows_filtered++;
return false;
}
std::vector<Slice> values;
- {
- split_line(line, &values);
- }
+ { split_line(line, &values); }
// range of current file
const TBrokerRangeDesc& range = _ranges.at(_next_range - 1);
@@ -421,19 +403,17 @@ bool BrokerScanner::line_to_src_tuple(const Slice& line) {
if (values.size() + columns_from_path.size() < _src_slot_descs.size()) {
std::stringstream error_msg;
error_msg << "actual column number is less than schema column number. "
- << "actual number: " << values.size() << " sep: " << _value_separator << ", "
- << "schema number: " << _src_slot_descs.size() << "; ";
- _state->append_error_msg_to_file(std::string(line.data, line.size),
- error_msg.str());
+ << "actual number: " << values.size() << " sep: " << _value_separator << ", "
+ << "schema number: " << _src_slot_descs.size() << "; ";
+ _state->append_error_msg_to_file(std::string(line.data, line.size), error_msg.str());
_counter->num_rows_filtered++;
return false;
} else if (values.size() + columns_from_path.size() > _src_slot_descs.size()) {
std::stringstream error_msg;
error_msg << "actual column number is more than schema column number. "
- << "actual number: " << values.size() << " sep: " << _value_separator << ", "
- << "schema number: " << _src_slot_descs.size() << "; ";
- _state->append_error_msg_to_file(std::string(line.data, line.size),
- error_msg.str());
+ << "actual number: " << values.size() << " sep: " << _value_separator << ", "
+ << "schema number: " << _src_slot_descs.size() << "; ";
+ _state->append_error_msg_to_file(std::string(line.data, line.size), error_msg.str());
_counter->num_rows_filtered++;
return false;
}
@@ -459,4 +439,4 @@ bool BrokerScanner::line_to_src_tuple(const Slice& line) {
return true;
}
-}
+} // namespace doris
diff --git a/be/src/exec/broker_scanner.h b/be/src/exec/broker_scanner.h
index 1eb1122..0c02baf 100644
--- a/be/src/exec/broker_scanner.h
+++ b/be/src/exec/broker_scanner.h
@@ -17,19 +17,19 @@
#pragma once
-#include <memory>
-#include <vector>
-#include <string>
#include <map>
+#include <memory>
#include <sstream>
+#include <string>
+#include <vector>
-#include "exec/base_scanner.h"
#include "common/status.h"
+#include "exec/base_scanner.h"
#include "gen_cpp/PlanNodes_types.h"
#include "gen_cpp/Types_types.h"
#include "runtime/mem_pool.h"
-#include "util/slice.h"
#include "util/runtime_profile.h"
+#include "util/slice.h"
namespace doris {
@@ -52,13 +52,9 @@ class StreamLoadPipe;
// Broker scanner convert the data read from broker to doris's tuple.
class BrokerScanner : public BaseScanner {
public:
- BrokerScanner(
- RuntimeState* state,
- RuntimeProfile* profile,
- const TBrokerScanRangeParams& params,
- const std::vector<TBrokerRangeDesc>& ranges,
- const std::vector<TNetworkAddress>& broker_addresses,
- ScannerCounter* counter);
+ BrokerScanner(RuntimeState* state, RuntimeProfile* profile,
+ const TBrokerScanRangeParams& params, const std::vector<TBrokerRangeDesc>& ranges,
+ const std::vector<TNetworkAddress>& broker_addresses, ScannerCounter* counter);
~BrokerScanner();
// Open this scanner, will initialize information need to
@@ -78,17 +74,13 @@ private:
Status open_next_reader();
// Split one text line to values
- void split_line(
- const Slice& line, std::vector<Slice>* values);
+ void split_line(const Slice& line, std::vector<Slice>* values);
- void fill_fix_length_string(
- const Slice& value, MemPool* pool,
- char** new_value_p, int new_value_length);
+ void fill_fix_length_string(const Slice& value, MemPool* pool, char** new_value_p,
+ int new_value_length);
- bool check_decimal_input(
- const Slice& value,
- int precision, int scale,
- std::stringstream* error_msg);
+ bool check_decimal_input(const Slice& value, int precision, int scale,
+ std::stringstream* error_msg);
// Convert one row to one tuple
// 'ptr' and 'len' is csv text line
@@ -99,7 +91,9 @@ private:
Status line_to_src_tuple();
bool line_to_src_tuple(const Slice& line);
-private:;
+
+private:
+ ;
const std::vector<TBrokerRangeDesc>& _ranges;
const std::vector<TNetworkAddress>& _broker_addresses;
@@ -125,4 +119,4 @@ private:;
std::shared_ptr<StreamLoadPipe> _stream_load_pipe;
};
-}
+} // namespace doris
diff --git a/be/src/exec/broker_writer.cpp b/be/src/exec/broker_writer.cpp
index 4474cd0..40a711a 100644
--- a/be/src/exec/broker_writer.cpp
+++ b/be/src/exec/broker_writer.cpp
@@ -29,20 +29,16 @@
namespace doris {
-BrokerWriter::BrokerWriter(
- ExecEnv* env,
- const std::vector<TNetworkAddress>& broker_addresses,
- const std::map<std::string, std::string>& properties,
- const std::string& path,
- int64_t start_offset) :
- _env(env),
- _addresses(broker_addresses),
- _properties(properties),
- _path(path),
- _cur_offset(start_offset),
- _is_closed(false),
- _addr_idx(0) {
-}
+BrokerWriter::BrokerWriter(ExecEnv* env, const std::vector<TNetworkAddress>& broker_addresses,
+ const std::map<std::string, std::string>& properties,
+ const std::string& path, int64_t start_offset)
+ : _env(env),
+ _addresses(broker_addresses),
+ _properties(properties),
+ _path(path),
+ _cur_offset(start_offset),
+ _is_closed(false),
+ _addr_idx(0) {}
BrokerWriter::~BrokerWriter() {
close();
@@ -79,7 +75,7 @@ Status BrokerWriter::open() {
request.__set_properties(_properties);
VLOG_ROW << "debug: send broker open writer request: "
- << apache::thrift::ThriftDebugString(request).c_str();
+ << apache::thrift::ThriftDebugString(request).c_str();
TBrokerOpenWriterResponse response;
try {
@@ -87,8 +83,7 @@ Status BrokerWriter::open() {
BrokerServiceConnection client(client_cache(_env), broker_addr, 10000, &status);
if (!status.ok()) {
LOG(WARNING) << "Create broker writer client failed. "
- << "broker=" << broker_addr
- << ", status=" << status.get_error_msg();
+ << "broker=" << broker_addr << ", status=" << status.get_error_msg();
return status;
}
@@ -106,12 +101,12 @@ Status BrokerWriter::open() {
}
VLOG_ROW << "debug: send broker open writer response: "
- << apache::thrift::ThriftDebugString(response).c_str();
+ << apache::thrift::ThriftDebugString(response).c_str();
if (response.opStatus.statusCode != TBrokerOperationStatusCode::OK) {
std::stringstream ss;
ss << "Open broker writer failed, broker:" << broker_addr
- << " failed:" << response.opStatus.message;
+ << " failed:" << response.opStatus.message;
LOG(WARNING) << ss.str();
return Status::InternalError(ss.str());
}
@@ -134,7 +129,7 @@ Status BrokerWriter::write(const uint8_t* buf, size_t buf_len, size_t* written_l
request.__set_data(std::string(reinterpret_cast<const char*>(buf), buf_len));
VLOG_ROW << "debug: send broker pwrite request: "
- << apache::thrift::ThriftDebugString(request).c_str();
+ << apache::thrift::ThriftDebugString(request).c_str();
TBrokerOperationStatus response;
try {
@@ -142,8 +137,7 @@ Status BrokerWriter::write(const uint8_t* buf, size_t buf_len, size_t* written_l
BrokerServiceConnection client(client_cache(_env), broker_addr, 10000, &status);
if (!status.ok()) {
LOG(WARNING) << "Create broker write client failed. "
- << "broker=" << broker_addr
- << ", status=" << status.get_error_msg();
+ << "broker=" << broker_addr << ", status=" << status.get_error_msg();
return status;
}
@@ -166,12 +160,11 @@ Status BrokerWriter::write(const uint8_t* buf, size_t buf_len, size_t* written_l
}
VLOG_ROW << "debug: send broker pwrite response: "
- << apache::thrift::ThriftDebugString(response).c_str();
+ << apache::thrift::ThriftDebugString(response).c_str();
if (response.statusCode != TBrokerOperationStatusCode::OK) {
std::stringstream ss;
- ss << "Fail to write to broker, broker:" << broker_addr
- << " msg:" << response.message;
+ ss << "Fail to write to broker, broker:" << broker_addr << " msg:" << response.message;
LOG(WARNING) << ss.str();
return Status::InternalError(ss.str());
}
@@ -192,7 +185,7 @@ Status BrokerWriter::close() {
request.__set_fd(_fd);
VLOG_ROW << "debug: send broker close writer request: "
- << apache::thrift::ThriftDebugString(request).c_str();
+ << apache::thrift::ThriftDebugString(request).c_str();
const TNetworkAddress& broker_addr = _addresses[_addr_idx];
TBrokerOperationStatus response;
@@ -203,7 +196,7 @@ Status BrokerWriter::close() {
BrokerServiceConnection client(client_cache(_env), broker_addr, 20000, &status);
if (!status.ok()) {
LOG(WARNING) << "Create broker write client failed. broker=" << broker_addr
- << ", status=" << status.get_error_msg();
+ << ", status=" << status.get_error_msg();
return status;
}
@@ -211,30 +204,28 @@ Status BrokerWriter::close() {
client->closeWriter(response, request);
} catch (apache::thrift::transport::TTransportException& e) {
LOG(WARNING) << "Close broker writer failed. broker=" << broker_addr
- << ", status=" << status.get_error_msg();
+ << ", status=" << status.get_error_msg();
status = client.reopen();
if (!status.ok()) {
LOG(WARNING) << "Reopen broker writer failed. broker=" << broker_addr
- << ", status=" << status.get_error_msg();
+ << ", status=" << status.get_error_msg();
return status;
}
client->closeWriter(response, request);
}
} catch (apache::thrift::TException& e) {
std::stringstream ss;
- ss << "Close broker writer failed, broker:" << broker_addr
- << " msg:" << e.what();
+ ss << "Close broker writer failed, broker:" << broker_addr << " msg:" << e.what();
LOG(WARNING) << ss.str();
return Status::InternalError(ss.str());
}
VLOG_ROW << "debug: send broker close writer response: "
- << apache::thrift::ThriftDebugString(response).c_str();
+ << apache::thrift::ThriftDebugString(response).c_str();
if (response.statusCode != TBrokerOperationStatusCode::OK) {
std::stringstream ss;
- ss << "Close broker writer failed, broker:" << broker_addr
- << " msg:" << response.message;
+ ss << "Close broker writer failed, broker:" << broker_addr << " msg:" << response.message;
LOG(WARNING) << ss.str();
return Status::InternalError(ss.str());
}
diff --git a/be/src/exec/broker_writer.h b/be/src/exec/broker_writer.h
index 1bb5b75..5858530 100644
--- a/be/src/exec/broker_writer.h
+++ b/be/src/exec/broker_writer.h
@@ -20,13 +20,13 @@
#include <stdint.h>
-#include <string>
#include <map>
+#include <string>
#include "common/status.h"
#include "exec/file_writer.h"
-#include "gen_cpp/Types_types.h"
#include "gen_cpp/PaloBrokerService_types.h"
+#include "gen_cpp/Types_types.h"
namespace doris {
@@ -37,11 +37,9 @@ class TNetworkAddress;
// Reader of broker file
class BrokerWriter : public FileWriter {
public:
- BrokerWriter(ExecEnv* env,
- const std::vector<TNetworkAddress>& broker_addresses,
- const std::map<std::string, std::string>& properties,
- const std::string& path,
- int64_t start_offset);
+ BrokerWriter(ExecEnv* env, const std::vector<TNetworkAddress>& broker_addresses,
+ const std::map<std::string, std::string>& properties, const std::string& path,
+ int64_t start_offset);
virtual ~BrokerWriter();
virtual Status open() override;
diff --git a/be/src/exec/buffered_reader.cpp b/be/src/exec/buffered_reader.cpp
index 696067f..96640b8 100644
--- a/be/src/exec/buffered_reader.cpp
+++ b/be/src/exec/buffered_reader.cpp
@@ -17,8 +17,8 @@
#include "exec/buffered_reader.h"
-#include <sstream>
#include <algorithm>
+#include <sstream>
#include "common/logging.h"
@@ -78,7 +78,8 @@ Status BufferedReader::readat(int64_t position, int64_t nbytes, int64_t* bytes_r
}
while (*bytes_read < nbytes) {
int64_t len;
- RETURN_IF_ERROR(_read_once(position + *bytes_read, nbytes - *bytes_read, &len, reinterpret_cast<char*>(out) + *bytes_read));
+ RETURN_IF_ERROR(_read_once(position + *bytes_read, nbytes - *bytes_read, &len,
+ reinterpret_cast<char*>(out) + *bytes_read));
// EOF
if (len <= 0) {
break;
@@ -88,7 +89,8 @@ Status BufferedReader::readat(int64_t position, int64_t nbytes, int64_t* bytes_r
return Status::OK();
}
-Status BufferedReader::_read_once(int64_t position, int64_t nbytes, int64_t* bytes_read, void* out) {
+Status BufferedReader::_read_once(int64_t position, int64_t nbytes, int64_t* bytes_read,
+ void* out) {
// requested bytes missed the local buffer
if (position >= _buffer_limit || position < _buffer_offset) {
// if requested length is larger than the capacity of buffer, do not
@@ -102,7 +104,7 @@ Status BufferedReader::_read_once(int64_t position, int64_t nbytes, int64_t* byt
*bytes_read = 0;
return Status::OK();
}
- }
+ }
int64_t len = std::min(_buffer_limit - position, nbytes);
int64_t off = position - _buffer_offset;
memcpy(out, _buffer + off, len);
@@ -149,4 +151,3 @@ bool BufferedReader::closed() {
}
} // namespace doris
-
diff --git a/be/src/exec/buffered_reader.h b/be/src/exec/buffered_reader.h
index d7f2fbd..c347ba4 100644
--- a/be/src/exec/buffered_reader.h
+++ b/be/src/exec/buffered_reader.h
@@ -20,14 +20,14 @@
#include <stdint.h>
#include "common/status.h"
-#include "olap/olap_define.h"
#include "exec/file_reader.h"
+#include "olap/olap_define.h"
namespace doris {
// Buffered Reader
-// Add a cache layer between the caller and the file reader to reduce the
-// times of calls to the read function to speed up.
+// Add a cache layer between the caller and the file reader to reduce the
+// times of calls to the read function to speed up.
class BufferedReader : public FileReader {
public:
// If the reader need the file size, set it when construct FileReader.
@@ -37,9 +37,10 @@ public:
virtual Status open() override;
- // Read
+ // Read
virtual Status read(uint8_t* buf, size_t* buf_len, bool* eof) override;
- virtual Status readat(int64_t position, int64_t nbytes, int64_t* bytes_read, void* out) override;
+ virtual Status readat(int64_t position, int64_t nbytes, int64_t* bytes_read,
+ void* out) override;
virtual Status read_one_message(uint8_t** buf, size_t* length) override;
virtual int64_t size() override;
virtual Status seek(int64_t position) override;
@@ -50,6 +51,7 @@ public:
private:
Status _fill();
Status _read_once(int64_t position, int64_t nbytes, int64_t* bytes_read, void* out);
+
private:
FileReader* _reader;
char* _buffer;
@@ -59,4 +61,4 @@ private:
int64_t _cur_offset;
};
-}
+} // namespace doris
diff --git a/be/src/exec/cross_join_node.cpp b/be/src/exec/cross_join_node.cpp
index c85877d..177df55 100644
--- a/be/src/exec/cross_join_node.cpp
+++ b/be/src/exec/cross_join_node.cpp
@@ -28,10 +28,8 @@
namespace doris {
-CrossJoinNode::CrossJoinNode(
- ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs)
- : BlockingJoinNode("CrossJoinNode", TJoinOp::CROSS_JOIN, pool, tnode, descs) {
-}
+CrossJoinNode::CrossJoinNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs)
+ : BlockingJoinNode("CrossJoinNode", TJoinOp::CROSS_JOIN, pool, tnode, descs) {}
Status CrossJoinNode::prepare(RuntimeState* state) {
DCHECK(_join_op == TJoinOp::CROSS_JOIN);
@@ -71,8 +69,7 @@ Status CrossJoinNode::construct_build_side(RuntimeState* state) {
SCOPED_TIMER(_build_timer);
_build_batches.add_row_batch(batch);
VLOG_ROW << build_list_debug_string();
- COUNTER_SET(_build_row_counter,
- static_cast<int64_t>(_build_batches.total_num_rows()));
+ COUNTER_SET(_build_row_counter, static_cast<int64_t>(_build_batches.total_num_rows()));
if (eos) {
break;
@@ -111,7 +108,7 @@ Status CrossJoinNode::get_next(RuntimeState* state, RowBatch* output_batch, bool
// Continue processing this row batch
_num_rows_returned +=
- process_left_child_batch(output_batch, _left_batch.get(), max_added_rows);
+ process_left_child_batch(output_batch, _left_batch.get(), max_added_rows);
COUNTER_SET(_rows_returned_counter, _num_rows_returned);
if (reached_limit() || output_batch->is_full()) {
@@ -153,7 +150,7 @@ std::string CrossJoinNode::build_list_debug_string() {
// TODO: this can be replaced with a codegen'd function
int CrossJoinNode::process_left_child_batch(RowBatch* output_batch, RowBatch* batch,
- int max_added_rows) {
+ int max_added_rows) {
int row_idx = output_batch->add_rows(max_added_rows);
DCHECK(row_idx != RowBatch::INVALID_ROW_INDEX);
uint8_t* output_row_mem = reinterpret_cast<uint8_t*>(output_batch->get_row(row_idx));
@@ -200,4 +197,4 @@ int CrossJoinNode::process_left_child_batch(RowBatch* output_batch, RowBatch* ba
output_batch->commit_rows(rows_returned);
return rows_returned;
}
-}
+} // namespace doris
diff --git a/be/src/exec/cross_join_node.h b/be/src/exec/cross_join_node.h
index 797a86c..54cb4fb 100644
--- a/be/src/exec/cross_join_node.h
+++ b/be/src/exec/cross_join_node.h
@@ -19,16 +19,16 @@
#define DORIS_BE_SRC_QUERY_EXEC_CROSS_JOIN_NODE_H
#include <boost/scoped_ptr.hpp>
-#include <boost/unordered_set.hpp>
#include <boost/thread.hpp>
+#include <boost/unordered_set.hpp>
#include <string>
-#include "exec/exec_node.h"
#include "exec/blocking_join_node.h"
+#include "exec/exec_node.h"
#include "exec/row_batch_list.h"
+#include "gen_cpp/PlanNodes_types.h"
#include "runtime/descriptors.h"
#include "runtime/mem_pool.h"
-#include "gen_cpp/PlanNodes_types.h"
namespace doris {
@@ -73,6 +73,6 @@ private:
std::string build_list_debug_string();
};
-}
+} // namespace doris
#endif
diff --git a/be/src/exec/csv_scan_node.cpp b/be/src/exec/csv_scan_node.cpp
index 58baecc..7b073ed 100644
--- a/be/src/exec/csv_scan_node.cpp
+++ b/be/src/exec/csv_scan_node.cpp
@@ -17,71 +17,57 @@
#include "csv_scan_node.h"
+#include <thrift/protocol/TDebugProtocol.h>
+
#include <string>
#include <vector>
-#include <thrift/protocol/TDebugProtocol.h>
-
#include "exec/text_converter.hpp"
#include "exprs/hll_hash_function.h"
#include "gen_cpp/PlanNodes_types.h"
-#include "runtime/runtime_state.h"
+#include "olap/olap_common.h"
+#include "olap/utils.h"
#include "runtime/row_batch.h"
+#include "runtime/runtime_state.h"
#include "runtime/string_value.h"
#include "runtime/tuple_row.h"
-#include "util/file_utils.h"
-#include "util/runtime_profile.h"
#include "util/debug_util.h"
+#include "util/file_utils.h"
#include "util/hash_util.hpp"
-#include "olap/olap_common.h"
-#include "olap/utils.h"
+#include "util/runtime_profile.h"
namespace doris {
class StringRef {
public:
- StringRef(char const* const begin, int const size) :
- _begin(begin), _size(size) {
- }
+ StringRef(char const* const begin, int const size) : _begin(begin), _size(size) {}
~StringRef() {
// No need to delete _begin, because it only record the index in a std::string.
// The c-string will be released along with the std::string object.
}
- int size() const {
- return _size;
- }
- int length() const {
- return _size;
- }
+ int size() const { return _size; }
+ int length() const { return _size; }
- char const* c_str() const {
- return _begin;
- }
- char const* begin() const {
- return _begin;
- }
+ char const* c_str() const { return _begin; }
+ char const* begin() const { return _begin; }
+
+ char const* end() const { return _begin + _size; }
- char const* end() const {
- return _begin + _size;
- }
private:
char const* _begin;
int _size;
};
void split_line(const std::string& str, char delimiter, std::vector<StringRef>& result) {
- enum State {
- IN_DELIM = 1,
- IN_TOKEN = 0
- };
+ enum State { IN_DELIM = 1, IN_TOKEN = 0 };
// line-begin char and line-end char are considered to be 'delimeter'
State state = IN_DELIM;
- char const* p_begin = str.c_str(); // Begin of either a token or a delimiter
+ char const* p_begin = str.c_str(); // Begin of either a token or a delimiter
for (string::const_iterator it = str.begin(); it != str.end(); ++it) {
- State const new_state = (*it == delimiter? IN_DELIM : IN_TOKEN);
+ State const new_state = (*it == delimiter ? IN_DELIM : IN_TOKEN);
if (new_state != state) {
if (new_state == IN_DELIM) {
result.push_back(StringRef(p_begin, &*it - p_begin));
@@ -98,24 +84,21 @@ void split_line(const std::string& str, char delimiter, std::vector<StringRef>&
result.push_back(StringRef(p_begin, (&*str.end() - p_begin) - state));
}
-CsvScanNode::CsvScanNode(
- ObjectPool* pool,
- const TPlanNode& tnode,
- const DescriptorTbl& descs) :
- ScanNode(pool, tnode, descs),
- _tuple_id(tnode.csv_scan_node.tuple_id),
- _file_paths(tnode.csv_scan_node.file_paths),
- _column_separator(tnode.csv_scan_node.column_separator),
- _column_type_map(tnode.csv_scan_node.column_type_mapping),
- _column_function_map(tnode.csv_scan_node.column_function_mapping),
- _columns(tnode.csv_scan_node.columns),
- _unspecified_columns(tnode.csv_scan_node.unspecified_columns),
- _default_values(tnode.csv_scan_node.default_values),
- _is_init(false),
- _tuple_desc(nullptr),
- _tuple_pool(nullptr),
- _text_converter(nullptr),
- _hll_column_num(0) {
+CsvScanNode::CsvScanNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs)
+ : ScanNode(pool, tnode, descs),
+ _tuple_id(tnode.csv_scan_node.tuple_id),
+ _file_paths(tnode.csv_scan_node.file_paths),
+ _column_separator(tnode.csv_scan_node.column_separator),
+ _column_type_map(tnode.csv_scan_node.column_type_mapping),
+ _column_function_map(tnode.csv_scan_node.column_function_mapping),
+ _columns(tnode.csv_scan_node.columns),
+ _unspecified_columns(tnode.csv_scan_node.unspecified_columns),
+ _default_values(tnode.csv_scan_node.default_values),
+ _is_init(false),
+ _tuple_desc(nullptr),
+ _tuple_pool(nullptr),
+ _text_converter(nullptr),
+ _hll_column_num(0) {
// do nothing
LOG(INFO) << "csv scan node: " << apache::thrift::ThriftDebugString(tnode).c_str();
}
@@ -178,10 +161,8 @@ Status CsvScanNode::prepare(RuntimeState* state) {
}
// add 'unspecified_columns' which have default values
- if (_unspecified_columns.end() != std::find(
- _unspecified_columns.begin(),
- _unspecified_columns.end(),
- column_name)) {
+ if (_unspecified_columns.end() !=
+ std::find(_unspecified_columns.begin(), _unspecified_columns.end(), column_name)) {
_column_slot_map[column_name] = slot;
}
}
@@ -204,17 +185,17 @@ Status CsvScanNode::prepare(RuntimeState* state) {
}
// new one scanner
- _csv_scanner.reset(new(std::nothrow) CsvScanner(_file_paths));
+ _csv_scanner.reset(new (std::nothrow) CsvScanner(_file_paths));
if (_csv_scanner.get() == nullptr) {
return Status::InternalError("new a csv scanner failed.");
}
- _tuple_pool.reset(new(std::nothrow) MemPool(state->instance_mem_tracker().get()));
+ _tuple_pool.reset(new (std::nothrow) MemPool(state->instance_mem_tracker().get()));
if (_tuple_pool.get() == nullptr) {
return Status::InternalError("new a mem pool failed.");
}
- _text_converter.reset(new(std::nothrow) TextConverter('\\'));
+ _text_converter.reset(new (std::nothrow) TextConverter('\\'));
if (_text_converter.get() == nullptr) {
return Status::InternalError("new a text convertor failed.");
}
@@ -322,7 +303,7 @@ Status CsvScanNode::get_next(RuntimeState* state, RowBatch* row_batch, bool* eos
state->update_num_rows_load_total(_num_rows_load_total);
state->update_num_rows_load_filtered(_num_rows_load_filtered);
VLOG_ROW << "normal_row_number: " << state->num_rows_load_success()
- << "; error_row_number: " << state->num_rows_load_filtered() << std::endl;
+ << "; error_row_number: " << state->num_rows_load_filtered() << std::endl;
row_batch->tuple_data_pool()->acquire_data(_tuple_pool.get(), false);
@@ -354,7 +335,7 @@ Status CsvScanNode::close(RuntimeState* state) {
// Summary normal line and error line number info
std::stringstream summary_msg;
summary_msg << "error line: " << _num_rows_load_filtered
- << "; normal line: " << state->num_rows_load_success();
+ << "; normal line: " << state->num_rows_load_success();
state->append_error_msg_to_file("", summary_msg.str(), true);
}
@@ -375,9 +356,8 @@ Status CsvScanNode::set_scan_ranges(const std::vector<TScanRangeParams>& scan_ra
return Status::OK();
}
-void CsvScanNode::fill_fix_length_string(
- const char* value, const int value_length, MemPool* pool,
- char** new_value_p, const int new_value_length) {
+void CsvScanNode::fill_fix_length_string(const char* value, const int value_length, MemPool* pool,
+ char** new_value_p, const int new_value_length) {
if (new_value_length != 0 && value_length < new_value_length) {
DCHECK(pool != nullptr);
*new_value_p = reinterpret_cast<char*>(pool->allocate(new_value_length));
@@ -388,10 +368,10 @@ void CsvScanNode::fill_fix_length_string(
(*new_value_p)[i] = '\0';
}
VLOG_ROW << "Fill fix length string. "
- << "value: [" << std::string(value, value_length) << "]; "
- << "value_length: " << value_length << "; "
- << "*new_value_p: [" << *new_value_p << "]; "
- << "new value length: " << new_value_length << std::endl;
+ << "value: [" << std::string(value, value_length) << "]; "
+ << "value_length: " << value_length << "; "
+ << "*new_value_p: [" << *new_value_p << "]; "
+ << "new value length: " << new_value_length << std::endl;
}
}
@@ -399,15 +379,14 @@ void CsvScanNode::fill_fix_length_string(
// .123 1.23 123. -1.23
// ATTN: The decimal point and (for negative numbers) the "-" sign are not counted.
// like '.123', it will be regarded as '0.123', but it match decimal(3, 3)
-bool CsvScanNode::check_decimal_input(
- const char* value, const int value_length,
- const int precision, const int scale,
- std::stringstream* error_msg) {
+bool CsvScanNode::check_decimal_input(const char* value, const int value_length,
+ const int precision, const int scale,
+ std::stringstream* error_msg) {
if (value_length > (precision + 2)) {
(*error_msg) << "the length of decimal value is overflow. "
- << "precision in schema: (" << precision << ", " << scale << "); "
- << "value: [" << std::string(value, value_length) << "]; "
- << "str actual length: " << value_length << ";";
+ << "precision in schema: (" << precision << ", " << scale << "); "
+ << "value: [" << std::string(value, value_length) << "]; "
+ << "str actual length: " << value_length << ";";
return false;
}
@@ -435,7 +414,7 @@ bool CsvScanNode::check_decimal_input(
int value_int_len = 0;
int value_frac_len = 0;
value_int_len = point_index - begin_index;
- value_frac_len = end_index- point_index;
+ value_frac_len = end_index - point_index;
if (point_index == -1) {
// an int value: like 123
@@ -443,18 +422,16 @@ bool CsvScanNode::check_decimal_input(
value_frac_len = 0;
} else {
value_int_len = point_index - begin_index;
- value_frac_len = end_index- point_index;
+ value_frac_len = end_index - point_index;
}
if (value_int_len > (precision - scale)) {
- (*error_msg) << "the int part length longer than schema precision ["
- << precision << "]. "
- << "value [" << std::string(value, value_length) << "]. ";
+ (*error_msg) << "the int part length longer than schema precision [" << precision << "]. "
+ << "value [" << std::string(value, value_length) << "]. ";
return false;
} else if (value_frac_len > scale) {
- (*error_msg) << "the frac part length longer than schema scale ["
- << scale << "]. "
- << "value [" << std::string(value, value_length) << "]. ";
+ (*error_msg) << "the frac part length longer than schema scale [" << scale << "]. "
+ << "value [" << std::string(value, value_length) << "]. ";
return false;
}
return true;
@@ -465,17 +442,15 @@ static bool is_null(const char* value, int value_length) {
}
// Writes a slot in _tuple from an value containing text data.
-bool CsvScanNode::check_and_write_text_slot(
- const std::string& column_name, const TColumnType& column_type,
- const char* value, int value_length,
- const SlotDescriptor* slot, RuntimeState* state,
- std::stringstream* error_msg) {
-
+bool CsvScanNode::check_and_write_text_slot(const std::string& column_name,
+ const TColumnType& column_type, const char* value,
+ int value_length, const SlotDescriptor* slot,
+ RuntimeState* state, std::stringstream* error_msg) {
if (value_length == 0 && !slot->type().is_string_type()) {
(*error_msg) << "the length of input should not be 0. "
- << "column_name: " << column_name << "; "
- << "type: " << slot->type() << "; "
- << "input_str: [" << std::string(value, value_length) << "].";
+ << "column_name: " << column_name << "; "
+ << "type: " << slot->type() << "; "
+ << "input_str: [" << std::string(value, value_length) << "].";
return false;
}
@@ -485,16 +460,16 @@ bool CsvScanNode::check_and_write_text_slot(
return true;
} else {
(*error_msg) << "value cannot be null. column name: " << column_name
- << "; type: " << slot->type() << "; input_str: ["
- << std::string(value, value_length) << "].";
+ << "; type: " << slot->type() << "; input_str: ["
+ << std::string(value, value_length) << "].";
return false;
}
}
if (!slot->is_nullable() && is_null(value, value_length)) {
(*error_msg) << "value cannot be null. column name: " << column_name
- << "; type: " << slot->type() << "; input_str: ["
- << std::string(value, value_length) << "].";
+ << "; type: " << slot->type() << "; input_str: ["
+ << std::string(value, value_length) << "].";
return false;
}
@@ -506,17 +481,16 @@ bool CsvScanNode::check_and_write_text_slot(
int char_len = column_type.len;
if (slot->type().type != TYPE_HLL && value_length > char_len) {
(*error_msg) << "the length of input is too long than schema. "
- << "column_name: " << column_name << "; "
- << "input_str: [" << std::string(value, value_length) << "] "
- << "type: " << slot->type() << "; "
- << "schema length: " << char_len << "; "
- << "actual length: " << value_length << "; ";
+ << "column_name: " << column_name << "; "
+ << "input_str: [" << std::string(value, value_length) << "] "
+ << "type: " << slot->type() << "; "
+ << "schema length: " << char_len << "; "
+ << "actual length: " << value_length << "; ";
return false;
}
if (slot->type().type == TYPE_CHAR && value_length < char_len) {
- fill_fix_length_string(
- value, value_length, _tuple_pool.get(),
- &value_to_convert, char_len);
+ fill_fix_length_string(value, value_length, _tuple_pool.get(), &value_to_convert,
+ char_len);
value_to_convert_length = char_len;
}
} else if (slot->type().is_decimal_type()) {
@@ -528,13 +502,11 @@ bool CsvScanNode::check_and_write_text_slot(
}
}
-
- if (!_text_converter->write_slot(slot, _tuple, value_to_convert, value_to_convert_length,
- true, false, _tuple_pool.get())) {
- (*error_msg) << "convert csv string to "
- << slot->type() << " failed. "
- << "column_name: " << column_name << "; "
- << "input_str: [" << std::string(value, value_length) << "]; ";
+ if (!_text_converter->write_slot(slot, _tuple, value_to_convert, value_to_convert_length, true,
+ false, _tuple_pool.get())) {
+ (*error_msg) << "convert csv string to " << slot->type() << " failed. "
+ << "column_name: " << column_name << "; "
+ << "input_str: [" << std::string(value, value_length) << "]; ";
return false;
}
@@ -554,14 +526,14 @@ bool CsvScanNode::split_check_fill(const std::string& line, RuntimeState* state)
if (_hll_column_num == 0 && fields.size() < _columns.size()) {
error_msg << "actual column number is less than schema column number. "
- << "actual number: " << fields.size() << " ,"
- << "schema number: " << _columns.size() << "; ";
+ << "actual number: " << fields.size() << " ,"
+ << "schema number: " << _columns.size() << "; ";
_runtime_state->append_error_msg_to_file(line, error_msg.str());
return false;
} else if (_hll_column_num == 0 && fields.size() > _columns.size()) {
error_msg << "actual column number is more than schema column number. "
- << "actual number: " << fields.size() << " ,"
- << "schema number: " << _columns.size() << "; ";
+ << "actual number: " << fields.size() << " ,"
+ << "schema number: " << _columns.size() << "; ";
_runtime_state->append_error_msg_to_file(line, error_msg.str());
return false;
}
@@ -583,11 +555,8 @@ bool CsvScanNode::split_check_fill(const std::string& line, RuntimeState* state)
}
const TColumnType& column_type = _column_type_vec[i];
- bool flag = check_and_write_text_slot(
- column_name, column_type,
- fields[i].c_str(),
- fields[i].length(),
- slot, state, &error_msg);
+ bool flag = check_and_write_text_slot(column_name, column_type, fields[i].c_str(),
+ fields[i].length(), slot, state, &error_msg);
if (flag == false) {
_runtime_state->append_error_msg_to_file(line, error_msg.str());
@@ -611,11 +580,8 @@ bool CsvScanNode::split_check_fill(const std::string& line, RuntimeState* state)
}
const TColumnType& column_type = _unspecified_colomn_type_vec[i];
- bool flag = check_and_write_text_slot(
- column_name, column_type,
- _default_values[i].c_str(),
- _default_values[i].length(),
- slot, state, &error_msg);
+ bool flag = check_and_write_text_slot(column_name, column_type, _default_values[i].c_str(),
+ _default_values[i].length(), slot, state, &error_msg);
if (flag == false) {
_runtime_state->append_error_msg_to_file(line, error_msg.str());
@@ -624,8 +590,7 @@ bool CsvScanNode::split_check_fill(const std::string& line, RuntimeState* state)
}
for (std::map<std::string, TMiniLoadEtlFunction>::iterator iter = _column_function_map.begin();
- iter != _column_function_map.end();
- iter++) {
+ iter != _column_function_map.end(); iter++) {
TMiniLoadEtlFunction& function = iter->second;
const std::string& column_name = iter->first;
const SlotDescriptor* slot = _column_slot_map[column_name];
@@ -634,11 +599,8 @@ bool CsvScanNode::split_check_fill(const std::string& line, RuntimeState* state)
const char* src = fields[function.param_column_index].c_str();
int src_column_len = fields[function.param_column_index].length();
hll_hash(src, src_column_len, &column_string);
- bool flag = check_and_write_text_slot(
- column_name, column_type,
- column_string.c_str(),
- column_string.length(),
- slot, state, &error_msg);
+ bool flag = check_and_write_text_slot(column_name, column_type, column_string.c_str(),
+ column_string.length(), slot, state, &error_msg);
if (flag == false) {
_runtime_state->append_error_msg_to_file(line, error_msg.str());
return false;
@@ -649,9 +611,8 @@ bool CsvScanNode::split_check_fill(const std::string& line, RuntimeState* state)
}
bool CsvScanNode::check_hll_function(TMiniLoadEtlFunction& function) {
- if (function.function_name.empty()
- || function.function_name != "hll_hash"
- || function.param_column_index < 0) {
+ if (function.function_name.empty() || function.function_name != "hll_hash" ||
+ function.param_column_index < 0) {
return false;
}
return true;
@@ -676,4 +637,3 @@ void CsvScanNode::hll_hash(const char* src, int len, std::string* result) {
}
} // end namespace doris
-
diff --git a/be/src/exec/csv_scan_node.h b/be/src/exec/csv_scan_node.h
index d320df6..5f830e8 100644
--- a/be/src/exec/csv_scan_node.h
+++ b/be/src/exec/csv_scan_node.h
@@ -18,11 +18,10 @@
#ifndef DORIS_BE_SRC_QUERY_EXEC_CSV_SCAN_NODE_H
#define DORIS_BE_SRC_QUERY_EXEC_CSV_SCAN_NODE_H
+#include <boost/scoped_ptr.hpp>
#include <fstream>
#include <sstream>
-#include <boost/scoped_ptr.hpp>
-
#include "common/config.h"
#include "exec/csv_scanner.h"
#include "exec/scan_node.h"
@@ -64,25 +63,20 @@ public:
virtual void debug_string(int indentation_level, std::stringstream* out) const;
private:
- bool check_and_write_text_slot(
- const std::string& column_name, const TColumnType& column_type,
- const char* value, int value_length,
- const SlotDescriptor* slot, RuntimeState* state,
- std::stringstream* error_msg);
+ bool check_and_write_text_slot(const std::string& column_name, const TColumnType& column_type,
+ const char* value, int value_length, const SlotDescriptor* slot,
+ RuntimeState* state, std::stringstream* error_msg);
// split one line into fields, check every fields, fill every field into tuple
bool split_check_fill(const std::string& line, RuntimeState* state);
- void fill_fix_length_string(
- const char* value, int value_length, MemPool* pool,
- char** new_value, int new_value_length);
- bool check_decimal_input(
- const char* value, int value_length,
- int precision, int scale,
- std::stringstream* error_msg);
-
- void hll_hash(const char* src, int len, std::string* result);
-
+ void fill_fix_length_string(const char* value, int value_length, MemPool* pool,
+ char** new_value, int new_value_length);
+ bool check_decimal_input(const char* value, int value_length, int precision, int scale,
+ std::stringstream* error_msg);
+
+ void hll_hash(const char* src, int len, std::string* result);
+
bool check_hll_function(TMiniLoadEtlFunction& function);
// Tuple id resolved in prepare() to set _tuple_desc;
@@ -139,4 +133,3 @@ private:
} // end namespace doris
#endif // DORIS_BE_SRC_QUERY_EXEC_CSV_SCAN_NODE_H
-
diff --git a/be/src/exec/csv_scanner.cpp b/be/src/exec/csv_scanner.cpp
index e21bef7..5841ecb 100644
--- a/be/src/exec/csv_scanner.cpp
+++ b/be/src/exec/csv_scanner.cpp
@@ -20,73 +20,72 @@
#include <boost/algorithm/string.hpp>
namespace doris {
- CsvScanner::CsvScanner(const std::vector<std::string>& csv_file_paths) :
- _is_open(false),
- _file_paths(csv_file_paths),
- _current_file(nullptr),
- _current_file_idx(0){
- // do nothing
- }
+CsvScanner::CsvScanner(const std::vector<std::string>& csv_file_paths)
+ : _is_open(false),
+ _file_paths(csv_file_paths),
+ _current_file(nullptr),
+ _current_file_idx(0) {
+ // do nothing
+}
- CsvScanner::~CsvScanner() {
- // close file
- if (_current_file != nullptr) {
- if (_current_file->is_open()) {
- _current_file->close();
- }
- delete _current_file;
- _current_file = nullptr;
+CsvScanner::~CsvScanner() {
+ // close file
+ if (_current_file != nullptr) {
+ if (_current_file->is_open()) {
+ _current_file->close();
}
+ delete _current_file;
+ _current_file = nullptr;
}
+}
- Status CsvScanner::open() {
- VLOG(1) << "CsvScanner::Connect";
+Status CsvScanner::open() {
+ VLOG(1) << "CsvScanner::Connect";
- if (_is_open) {
- LOG(INFO) << "this scanner already opened";
- return Status::OK();
- }
+ if (_is_open) {
+ LOG(INFO) << "this scanner already opened";
+ return Status::OK();
+ }
- if (_file_paths.empty()) {
- return Status::InternalError("no file specified.");
- }
+ if (_file_paths.empty()) {
+ return Status::InternalError("no file specified.");
+ }
+
+ _is_open = true;
+ return Status::OK();
+}
- _is_open = true;
+// TODO(lingbin): read more than one line at a time to reduce IO comsumption
+Status CsvScanner::get_next_row(std::string* line_str, bool* eos) {
+ if (_current_file == nullptr && _current_file_idx == _file_paths.size()) {
+ *eos = true;
return Status::OK();
}
- // TODO(lingbin): read more than one line at a time to reduce IO comsumption
- Status CsvScanner::get_next_row(std::string* line_str, bool* eos) {
- if (_current_file == nullptr && _current_file_idx == _file_paths.size()) {
- *eos = true;
- return Status::OK();
- }
-
- if (_current_file == nullptr && _current_file_idx < _file_paths.size()) {
- std::string& file_path = _file_paths[_current_file_idx];
- LOG(INFO) << "open csv file: [" << _current_file_idx << "] " << file_path;
+ if (_current_file == nullptr && _current_file_idx < _file_paths.size()) {
+ std::string& file_path = _file_paths[_current_file_idx];
+ LOG(INFO) << "open csv file: [" << _current_file_idx << "] " << file_path;
- _current_file = new std::ifstream(file_path, std::ifstream::in);
- if (!_current_file->is_open()) {
- return Status::InternalError("Fail to read csv file: " + file_path);
- }
- ++_current_file_idx;
+ _current_file = new std::ifstream(file_path, std::ifstream::in);
+ if (!_current_file->is_open()) {
+ return Status::InternalError("Fail to read csv file: " + file_path);
}
+ ++_current_file_idx;
+ }
- getline(*_current_file, *line_str);
- if (_current_file->eof()) {
- _current_file->close();
- delete _current_file;
- _current_file = nullptr;
+ getline(*_current_file, *line_str);
+ if (_current_file->eof()) {
+ _current_file->close();
+ delete _current_file;
+ _current_file = nullptr;
- if (_current_file_idx == _file_paths.size()) {
- *eos = true;
- return Status::OK();
- }
+ if (_current_file_idx == _file_paths.size()) {
+ *eos = true;
+ return Status::OK();
}
-
- *eos = false;
- return Status::OK();
}
-} // end namespace doris
+ *eos = false;
+ return Status::OK();
+}
+} // end namespace doris
diff --git a/be/src/exec/csv_scanner.h b/be/src/exec/csv_scanner.h
index aaf5b4b..3de2fef 100644
--- a/be/src/exec/csv_scanner.h
+++ b/be/src/exec/csv_scanner.h
@@ -33,6 +33,7 @@ public:
Status open();
Status get_next_row(std::string* line_str, bool* eos);
+
private:
bool _is_open;
std::vector<std::string> _file_paths;
@@ -43,4 +44,3 @@ private:
} // end namespace doris
#endif // DORIS_BE_SRC_QUERY_EXEC_CSV_SCANNER_H
-
diff --git a/be/src/exec/data_sink.cpp b/be/src/exec/data_sink.cpp
index 50a27cb..47bea39 100644
--- a/be/src/exec/data_sink.cpp
+++ b/be/src/exec/data_sink.cpp
@@ -17,33 +17,31 @@
#include "exec/data_sink.h"
-#include <string>
#include <map>
#include <memory>
+#include <string>
#include "common/logging.h"
#include "exec/exec_node.h"
#include "exec/tablet_sink.h"
#include "exprs/expr.h"
#include "gen_cpp/PaloInternalService_types.h"
+#include "runtime/data_spliter.h"
#include "runtime/data_stream_sender.h"
-#include "runtime/result_sink.h"
+#include "runtime/export_sink.h"
#include "runtime/memory_scratch_sink.h"
#include "runtime/mysql_table_sink.h"
-#include "runtime/data_spliter.h"
-#include "runtime/export_sink.h"
+#include "runtime/result_sink.h"
#include "runtime/runtime_state.h"
#include "util/logging.h"
namespace doris {
-Status DataSink::create_data_sink(
- ObjectPool* pool,
- const TDataSink& thrift_sink,
- const std::vector<TExpr>& output_exprs,
- const TPlanFragmentExecParams& params,
- const RowDescriptor& row_desc,
- boost::scoped_ptr<DataSink>* sink) {
+Status DataSink::create_data_sink(ObjectPool* pool, const TDataSink& thrift_sink,
+ const std::vector<TExpr>& output_exprs,
+ const TPlanFragmentExecParams& params,
+ const RowDescriptor& row_desc,
+ boost::scoped_ptr<DataSink>* sink) {
DataSink* tmp_sink = NULL;
switch (thrift_sink.type) {
@@ -51,13 +49,14 @@ Status DataSink::create_data_sink(
if (!thrift_sink.__isset.stream_sink) {
return Status::InternalError("Missing data stream sink.");
}
- bool send_query_statistics_with_every_batch = params.__isset.send_query_statistics_with_every_batch ?
- params.send_query_statistics_with_every_batch : false;
+ bool send_query_statistics_with_every_batch =
+ params.__isset.send_query_statistics_with_every_batch
+ ? params.send_query_statistics_with_every_batch
+ : false;
// TODO: figure out good buffer size based on size of output row
- tmp_sink = new DataStreamSender(
- pool, params.sender_id, row_desc,
- thrift_sink.stream_sink, params.destinations, 16 * 1024,
- send_query_statistics_with_every_batch);
+ tmp_sink = new DataStreamSender(pool, params.sender_id, row_desc, thrift_sink.stream_sink,
+ params.destinations, 16 * 1024,
+ send_query_statistics_with_every_batch);
// RETURN_IF_ERROR(sender->prepare(state->obj_pool(), thrift_sink.stream_sink));
sink->reset(tmp_sink);
break;
@@ -86,12 +85,12 @@ Status DataSink::create_data_sink(
}
// TODO: figure out good buffer size based on size of output row
- MysqlTableSink* mysql_tbl_sink = new MysqlTableSink(
- pool, row_desc, output_exprs);
+ MysqlTableSink* mysql_tbl_sink = new MysqlTableSink(pool, row_desc, output_exprs);
sink->reset(mysql_tbl_sink);
break;
#else
- return Status::InternalError("Don't support MySQL table, you should rebuild Doris with WITH_MYSQL option ON");
+ return Status::InternalError(
+ "Don't support MySQL table, you should rebuild Doris with WITH_MYSQL option ON");
#endif
}
@@ -102,9 +101,7 @@ Status DataSink::create_data_sink(
// TODO: figure out good buffer size based on size of output row
std::unique_ptr<DataSpliter> data_spliter(new DataSpliter(row_desc));
- RETURN_IF_ERROR(DataSpliter::from_thrift(pool,
- thrift_sink.split_sink,
- data_spliter.get()));
+ RETURN_IF_ERROR(DataSpliter::from_thrift(pool, thrift_sink.split_sink, data_spliter.get()));
sink->reset(data_spliter.release());
break;
}
@@ -129,7 +126,7 @@ Status DataSink::create_data_sink(
default:
std::stringstream error_msg;
std::map<int, const char*>::const_iterator i =
- _TDataSinkType_VALUES_TO_NAMES.find(thrift_sink.type);
+ _TDataSinkType_VALUES_TO_NAMES.find(thrift_sink.type);
const char* str = "Unknown data sink type ";
if (i != _TDataSinkType_VALUES_TO_NAMES.end()) {
@@ -152,9 +149,10 @@ Status DataSink::init(const TDataSink& thrift_sink) {
}
Status DataSink::prepare(RuntimeState* state) {
- _expr_mem_tracker = MemTracker::CreateTracker(-1, std::string("DataSink:") + std::to_string(state->load_job_id()),
- state->instance_mem_tracker());
+ _expr_mem_tracker = MemTracker::CreateTracker(
+ -1, std::string("DataSink:") + std::to_string(state->load_job_id()),
+ state->instance_mem_tracker());
return Status::OK();
}
-} // namespace doris
+} // namespace doris
diff --git a/be/src/exec/data_sink.h b/be/src/exec/data_sink.h
index 1c26e2d..daf06ed 100644
--- a/be/src/exec/data_sink.h
+++ b/be/src/exec/data_sink.h
@@ -64,17 +64,17 @@ public:
// be ignored.
virtual Status close(RuntimeState* state, Status exec_status) {
_expr_mem_tracker.reset();
- _closed = true;
+ _closed = true;
return Status::OK();
}
// Creates a new data sink from thrift_sink. A pointer to the
// new sink is written to *sink, and is owned by the caller.
- static Status create_data_sink(
- ObjectPool* pool,
- const TDataSink& thrift_sink, const std::vector<TExpr>& output_exprs,
- const TPlanFragmentExecParams& params,
- const RowDescriptor& row_desc, boost::scoped_ptr<DataSink>* sink);
+ static Status create_data_sink(ObjectPool* pool, const TDataSink& thrift_sink,
+ const std::vector<TExpr>& output_exprs,
+ const TPlanFragmentExecParams& params,
+ const RowDescriptor& row_desc,
+ boost::scoped_ptr<DataSink>* sink);
// Returns the runtime profile for the sink.
virtual RuntimeProfile* profile() = 0;
@@ -82,6 +82,7 @@ public:
virtual void set_query_statistics(std::shared_ptr<QueryStatistics> statistics) {
_query_statistics = statistics;
}
+
protected:
// Set to true after close() has been called. subclasses should check and set this in
// close().
@@ -92,5 +93,5 @@ protected:
std::shared_ptr<QueryStatistics> _query_statistics;
};
-} // namespace doris
+} // namespace doris
#endif
diff --git a/be/src/exec/decompressor.cpp b/be/src/exec/decompressor.cpp
index c64a017..ea6cc0e 100644
--- a/be/src/exec/decompressor.cpp
+++ b/be/src/exec/decompressor.cpp
@@ -19,9 +19,8 @@
namespace doris {
-Status Decompressor::create_decompressor(CompressType type,
- Decompressor** decompressor) {
- switch(type) {
+Status Decompressor::create_decompressor(CompressType type, Decompressor** decompressor) {
+ switch (type) {
case CompressType::UNCOMPRESSED:
*decompressor = nullptr;
break;
@@ -52,25 +51,23 @@ Status Decompressor::create_decompressor(CompressType type,
if (*decompressor != nullptr) {
st = (*decompressor)->init();
}
-
+
return st;
}
-Decompressor::~Decompressor() {
-}
+Decompressor::~Decompressor() {}
std::string Decompressor::debug_info() {
return "Decompressor";
}
// Gzip
-GzipDecompressor::GzipDecompressor(bool is_deflate):
- Decompressor(_is_deflate ? CompressType::DEFLATE : CompressType::GZIP),
- _is_deflate(is_deflate) {
-}
+GzipDecompressor::GzipDecompressor(bool is_deflate)
+ : Decompressor(_is_deflate ? CompressType::DEFLATE : CompressType::GZIP),
+ _is_deflate(is_deflate) {}
GzipDecompressor::~GzipDecompressor() {
- (void) inflateEnd(&_z_strm);
+ (void)inflateEnd(&_z_strm);
}
Status GzipDecompressor::init() {
@@ -90,11 +87,10 @@ Status GzipDecompressor::init() {
return Status::OK();
}
-Status GzipDecompressor::decompress(
- uint8_t* input, size_t input_len, size_t* input_bytes_read,
- uint8_t* output, size_t output_max_len,
- size_t* decompressed_len, bool* stream_end,
- size_t* more_input_bytes, size_t* more_output_bytes) {
+Status GzipDecompressor::decompress(uint8_t* input, size_t input_len, size_t* input_bytes_read,
+ uint8_t* output, size_t output_max_len,
+ size_t* decompressed_len, bool* stream_end,
+ size_t* more_input_bytes, size_t* more_output_bytes) {
// 1. set input and output
_z_strm.next_in = input;
_z_strm.avail_in = input_len;
@@ -109,14 +105,13 @@ Status GzipDecompressor::decompress(
// Provide more output starting at next_out and update next_out and avail_out
// accordingly.
// inflate() returns Z_OK if some progress has been made (more input processed
- // or more output produced)
+ // or more output produced)
int ret = inflate(&_z_strm, Z_NO_FLUSH);
*input_bytes_read = input_len - _z_strm.avail_in;
*decompressed_len = output_max_len - _z_strm.avail_out;
- VLOG(10) << "gzip dec ret: " << ret
- << " input_bytes_read: " << *input_bytes_read
+ VLOG(10) << "gzip dec ret: " << ret << " input_bytes_read: " << *input_bytes_read
<< " decompressed_len: " << *decompressed_len;
if (ret == Z_BUF_ERROR) {
@@ -150,7 +145,8 @@ Status GzipDecompressor::decompress(
std::string GzipDecompressor::debug_info() {
std::stringstream ss;
- ss << "GzipDecompressor." << " is_deflate: " << _is_deflate;
+ ss << "GzipDecompressor."
+ << " is_deflate: " << _is_deflate;
return ss.str();
}
@@ -171,12 +167,10 @@ Status Bzip2Decompressor::init() {
return Status::OK();
}
-Status Bzip2Decompressor::decompress(
- uint8_t* input, size_t input_len, size_t* input_bytes_read,
- uint8_t* output, size_t output_max_len,
- size_t* decompressed_len, bool* stream_end,
- size_t* more_input_bytes, size_t* more_output_bytes) {
-
+Status Bzip2Decompressor::decompress(uint8_t* input, size_t input_len, size_t* input_bytes_read,
+ uint8_t* output, size_t output_max_len,
+ size_t* decompressed_len, bool* stream_end,
+ size_t* more_input_bytes, size_t* more_output_bytes) {
// 1. set input and output
_bz_strm.next_in = const_cast<char*>(reinterpret_cast<const char*>(input));
_bz_strm.avail_in = input_len;
@@ -252,17 +246,15 @@ Status Lz4FrameDecompressor::init() {
return Status::OK();
}
-Status Lz4FrameDecompressor::decompress(
- uint8_t* input, size_t input_len, size_t* input_bytes_read,
- uint8_t* output, size_t output_max_len,
- size_t* decompressed_len, bool* stream_end,
- size_t* more_input_bytes, size_t* more_output_bytes) {
-
+Status Lz4FrameDecompressor::decompress(uint8_t* input, size_t input_len, size_t* input_bytes_read,
+ uint8_t* output, size_t output_max_len,
+ size_t* decompressed_len, bool* stream_end,
+ size_t* more_input_bytes, size_t* more_output_bytes) {
uint8_t* src = input;
size_t src_size = input_len;
size_t ret = 1;
*input_bytes_read = 0;
-
+
if (_expect_dec_buf_size == -1) {
// init expected decompress buf size, and check if output_max_len is large enough
// ATTN: _expect_dec_buf_size is uninit, which means this is the first time to call
@@ -277,7 +269,7 @@ Status Lz4FrameDecompressor::decompress(
}
LZ4F_frameInfo_t info;
- ret = LZ4F_getFrameInfo(_dctx, &info, (void*) src, &src_size);
+ ret = LZ4F_getFrameInfo(_dctx, &info, (void*)src, &src_size);
if (LZ4F_isError(ret)) {
std::stringstream ss;
ss << "LZ4F_getFrameInfo error: " << std::string(LZ4F_getErrorName(ret));
@@ -298,11 +290,11 @@ Status Lz4FrameDecompressor::decompress(
src_size = input_len - src_size;
LOG(INFO) << "lz4 block size: " << _expect_dec_buf_size;
- }
-
+ }
+
// decompress
size_t output_len = output_max_len;
- ret = LZ4F_decompress(_dctx, (void*) output, &output_len, (void*) src, &src_size,
+ ret = LZ4F_decompress(_dctx, (void*)output, &output_len, (void*)src, &src_size,
/* LZ4F_decompressOptions_t */ NULL);
if (LZ4F_isError(ret)) {
std::stringstream ss;
@@ -332,15 +324,19 @@ std::string Lz4FrameDecompressor::debug_info() {
size_t Lz4FrameDecompressor::get_block_size(const LZ4F_frameInfo_t* info) {
switch (info->blockSizeID) {
- case LZ4F_default:
- case LZ4F_max64KB: return 1 << 16;
- case LZ4F_max256KB: return 1 << 18;
- case LZ4F_max1MB: return 1 << 20;
- case LZ4F_max4MB: return 1 << 22;
- default:
- // error
- return -1;
+ case LZ4F_default:
+ case LZ4F_max64KB:
+ return 1 << 16;
+ case LZ4F_max256KB:
+ return 1 << 18;
+ case LZ4F_max1MB:
+ return 1 << 20;
+ case LZ4F_max4MB:
+ return 1 << 22;
+ default:
+ // error
+ return -1;
}
}
-} // namespace
+} // namespace doris
diff --git a/be/src/exec/decompressor.h b/be/src/exec/decompressor.h
index 9f81c18..08228da 100644
--- a/be/src/exec/decompressor.h
+++ b/be/src/exec/decompressor.h
@@ -17,27 +17,20 @@
#pragma once
-#include <zlib.h>
#include <bzlib.h>
#include <lz4/lz4frame.h>
+#include <zlib.h>
#ifdef DORIS_WITH_LZO
-#include <lzo/lzoconf.h>
#include <lzo/lzo1x.h>
+#include <lzo/lzoconf.h>
#endif
#include "common/status.h"
namespace doris {
-enum CompressType {
- UNCOMPRESSED,
- GZIP,
- DEFLATE,
- BZIP2,
- LZ4FRAME,
- LZOP
-};
+enum CompressType { UNCOMPRESSED, GZIP, DEFLATE, BZIP2, LZ4FRAME, LZOP };
class Decompressor {
public:
@@ -55,16 +48,14 @@ public:
// more_input_bytes(out): decompressor need more bytes to consume
// more_output_bytes(out): decompressor need more space to save decompressed data
//
- // input and output buf should be allocated and released outside
- virtual Status decompress(
- uint8_t* input, size_t input_len, size_t* input_bytes_read,
- uint8_t* output, size_t output_max_len,
- size_t* decompressed_len, bool* stream_end,
- size_t* more_input_bytes, size_t* more_output_bytes) = 0;
+ // input and output buf should be allocated and released outside
+ virtual Status decompress(uint8_t* input, size_t input_len, size_t* input_bytes_read,
+ uint8_t* output, size_t output_max_len, size_t* decompressed_len,
+ bool* stream_end, size_t* more_input_bytes,
+ size_t* more_output_bytes) = 0;
public:
- static Status create_decompressor(CompressType type,
- Decompressor** decompressor);
+ static Status create_decompressor(CompressType type, Decompressor** decompressor);
virtual std::string debug_info();
@@ -73,7 +64,7 @@ public:
protected:
virtual Status init() = 0;
- Decompressor(CompressType ctype):_ctype(ctype) {}
+ Decompressor(CompressType ctype) : _ctype(ctype) {}
CompressType _ctype;
};
@@ -82,11 +73,10 @@ class GzipDecompressor : public Decompressor {
public:
virtual ~GzipDecompressor();
- virtual Status decompress(
- uint8_t* input, size_t input_len, size_t* input_bytes_read,
- uint8_t* output, size_t output_max_len,
- size_t* decompressed_len, bool* stream_end,
- size_t* more_input_bytes, size_t* more_output_bytes) override;
+ virtual Status decompress(uint8_t* input, size_t input_len, size_t* input_bytes_read,
+ uint8_t* output, size_t output_max_len, size_t* decompressed_len,
+ bool* stream_end, size_t* more_input_bytes,
+ size_t* more_output_bytes) override;
virtual std::string debug_info() override;
@@ -100,20 +90,19 @@ private:
z_stream _z_strm;
- // These are magic numbers from zlib.h. Not clear why they are not defined there.
- const static int WINDOW_BITS = 15; // Maximum window size
- const static int DETECT_CODEC = 32; // Determine if this is libz or gzip from header.
+ // These are magic numbers from zlib.h. Not clear why they are not defined there.
+ const static int WINDOW_BITS = 15; // Maximum window size
+ const static int DETECT_CODEC = 32; // Determine if this is libz or gzip from header.
};
class Bzip2Decompressor : public Decompressor {
public:
virtual ~Bzip2Decompressor();
- virtual Status decompress(
- uint8_t* input, size_t input_len, size_t* input_bytes_read,
- uint8_t* output, size_t output_max_len,
- size_t* decompressed_len, bool* stream_end,
- size_t* more_input_bytes, size_t* more_output_bytes) override;
+ virtual Status decompress(uint8_t* input, size_t input_len, size_t* input_bytes_read,
+ uint8_t* output, size_t output_max_len, size_t* decompressed_len,
+ bool* stream_end, size_t* more_input_bytes,
+ size_t* more_output_bytes) override;
virtual std::string debug_info() override;
@@ -130,11 +119,10 @@ class Lz4FrameDecompressor : public Decompressor {
public:
virtual ~Lz4FrameDecompressor();
- virtual Status decompress(
- uint8_t* input, size_t input_len, size_t* input_bytes_read,
- uint8_t* output, size_t output_max_len,
- size_t* decompressed_len, bool* stream_end,
- size_t* more_input_bytes, size_t* more_output_bytes) override;
+ virtual Status decompress(uint8_t* input, size_t input_len, size_t* input_bytes_read,
+ uint8_t* output, size_t output_max_len, size_t* decompressed_len,
+ bool* stream_end, size_t* more_input_bytes,
+ size_t* more_output_bytes) override;
virtual std::string debug_info() override;
@@ -156,28 +144,21 @@ class LzopDecompressor : public Decompressor {
public:
virtual ~LzopDecompressor();
- virtual Status decompress(
- uint8_t* input, size_t input_len, size_t* input_bytes_read,
- uint8_t* output, size_t output_max_len,
- size_t* decompressed_len, bool* stream_end,
- size_t* more_input_bytes, size_t* more_output_bytes) override;
+ virtual Status decompress(uint8_t* input, size_t input_len, size_t* input_bytes_read,
+ uint8_t* output, size_t output_max_len, size_t* decompressed_len,
+ bool* stream_end, size_t* more_input_bytes,
+ size_t* more_output_bytes) override;
virtual std::string debug_info() override;
private:
friend class Decompressor;
- LzopDecompressor() :
- Decompressor(CompressType::LZOP),
- _header_info({0}),
- _is_header_loaded(false) {}
+ LzopDecompressor()
+ : Decompressor(CompressType::LZOP), _header_info({0}), _is_header_loaded(false) {}
virtual Status init() override;
private:
- enum LzoChecksum {
- CHECK_NONE,
- CHECK_CRC32,
- CHECK_ADLER
- };
+ enum LzoChecksum { CHECK_NONE, CHECK_CRC32, CHECK_ADLER };
private:
inline uint8_t* get_uint8(uint8_t* ptr, uint8_t* value) {
@@ -200,22 +181,18 @@ private:
}
inline LzoChecksum input_type(int flags) {
- return (flags & F_CRC32_C) ? CHECK_CRC32 :
- (flags & F_ADLER32_C) ? CHECK_ADLER : CHECK_NONE;
+ return (flags & F_CRC32_C) ? CHECK_CRC32 : (flags & F_ADLER32_C) ? CHECK_ADLER : CHECK_NONE;
}
inline LzoChecksum output_type(int flags) {
- return (flags & F_CRC32_D) ? CHECK_CRC32 :
- (flags & F_ADLER32_D) ? CHECK_ADLER : CHECK_NONE;
+ return (flags & F_CRC32_D) ? CHECK_CRC32 : (flags & F_ADLER32_D) ? CHECK_ADLER : CHECK_NONE;
}
- Status parse_header_info(uint8_t* input, size_t input_len,
- size_t* input_bytes_read,
+ Status parse_header_info(uint8_t* input, size_t input_len, size_t* input_bytes_read,
size_t* more_bytes_needed);
- Status checksum(LzoChecksum type, const std::string& source,
- uint32_t expected,
- uint8_t* ptr, size_t len);
+ Status checksum(LzoChecksum type, const std::string& source, uint32_t expected, uint8_t* ptr,
+ size_t len);
private:
// lzop header info
@@ -259,6 +236,6 @@ private:
const static uint64_t F_CRC32_D;
const static uint64_t F_ADLER32_D;
};
-#endif // DORIS_WITH_LZO
+#endif // DORIS_WITH_LZO
-} // namespace
+} // namespace doris
diff --git a/be/src/exec/empty_set_node.cpp b/be/src/exec/empty_set_node.cpp
index 4284d63..b03ff33 100644
--- a/be/src/exec/empty_set_node.cpp
+++ b/be/src/exec/empty_set_node.cpp
@@ -19,15 +19,12 @@
namespace doris {
-EmptySetNode::EmptySetNode(ObjectPool* pool, const TPlanNode& tnode,
- const DescriptorTbl& descs)
- : ExecNode(pool, tnode, descs) {
-}
+EmptySetNode::EmptySetNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs)
+ : ExecNode(pool, tnode, descs) {}
Status EmptySetNode::get_next(RuntimeState* state, RowBatch* row_batch, bool* eos) {
- *eos = true;
- return Status::OK();
-}
-
+ *eos = true;
+ return Status::OK();
}
+} // namespace doris
diff --git a/be/src/exec/empty_set_node.h b/be/src/exec/empty_set_node.h
index 93e5243..d31dd2e 100644
--- a/be/src/exec/empty_set_node.h
+++ b/be/src/exec/empty_set_node.h
@@ -29,5 +29,4 @@ public:
virtual Status get_next(RuntimeState* state, RowBatch* row_batch, bool* eos) override;
};
-}
-
+} // namespace doris
diff --git a/be/src/exec/es/es_predicate.cpp b/be/src/exec/es/es_predicate.cpp
index 1070b45..b455814 100644
--- a/be/src/exec/es/es_predicate.cpp
+++ b/be/src/exec/es/es_predicate.cpp
@@ -17,30 +17,29 @@
#include "exec/es/es_predicate.h"
+#include <gutil/strings/substitute.h>
#include <stdint.h>
+
+#include <boost/algorithm/string.hpp>
#include <map>
#include <sstream>
-#include <boost/algorithm/string.hpp>
-#include <gutil/strings/substitute.h>
-#include "common/status.h"
#include "common/logging.h"
+#include "common/status.h"
#include "exec/es/es_query_builder.h"
#include "exprs/expr.h"
#include "exprs/expr_context.h"
#include "exprs/in_predicate.h"
-
#include "gen_cpp/PlanNodes_types.h"
#include "olap/olap_common.h"
#include "olap/utils.h"
#include "runtime/client_cache.h"
-#include "runtime/runtime_state.h"
-#include "runtime/row_batch.h"
#include "runtime/datetime_value.h"
#include "runtime/large_int_value.h"
+#include "runtime/row_batch.h"
+#include "runtime/runtime_state.h"
#include "runtime/string_value.h"
#include "runtime/tuple_row.h"
-
#include "service/backend_options.h"
#include "util/debug_util.h"
#include "util/runtime_profile.h"
@@ -49,64 +48,63 @@ namespace doris {
using namespace std;
-#define RETURN_ERROR_IF_EXPR_IS_NOT_SLOTREF(expr) \
- do { \
- const Expr* expr_without_cast = Expr::expr_without_cast(expr); \
- if (expr_without_cast->node_type() != TExprNodeType::SLOT_REF) { \
- return Status::InternalError("build disjuncts failed: child is not slot ref"); \
- } \
+#define RETURN_ERROR_IF_EXPR_IS_NOT_SLOTREF(expr) \
+ do { \
+ const Expr* expr_without_cast = Expr::expr_without_cast(expr); \
+ if (expr_without_cast->node_type() != TExprNodeType::SLOT_REF) { \
+ return Status::InternalError("build disjuncts failed: child is not slot ref"); \
+ } \
} while (false)
std::string ExtLiteral::value_to_string() {
std::stringstream ss;
switch (_type) {
- case TYPE_TINYINT:
- ss << std::to_string(get_byte());
- break;
- case TYPE_SMALLINT:
- ss << std::to_string(get_short());
- break;
- case TYPE_INT:
- ss << std::to_string(get_int());
- break;
- case TYPE_BIGINT:
- ss << std::to_string(get_long());
- break;
- case TYPE_FLOAT:
- ss << std::to_string(get_float());
- break;
- case TYPE_DOUBLE:
- ss << std::to_string(get_double());
- break;
- case TYPE_CHAR:
- case TYPE_VARCHAR:
- ss << get_string();
- break;
- case TYPE_DATE:
- case TYPE_DATETIME:
- ss << get_date_string();
- break;
- case TYPE_BOOLEAN:
- ss << std::to_string(get_bool());
- break;
- case TYPE_DECIMAL:
- ss << get_decimal_string();
- break;
- case TYPE_DECIMALV2:
- ss << get_decimalv2_string();
- break;
- case TYPE_LARGEINT:
- ss << get_largeint_string();
- break;
- default:
- DCHECK(false);
- break;
+ case TYPE_TINYINT:
+ ss << std::to_string(get_byte());
+ break;
+ case TYPE_SMALLINT:
+ ss << std::to_string(get_short());
+ break;
+ case TYPE_INT:
+ ss << std::to_string(get_int());
+ break;
+ case TYPE_BIGINT:
+ ss << std::to_string(get_long());
+ break;
+ case TYPE_FLOAT:
+ ss << std::to_string(get_float());
+ break;
+ case TYPE_DOUBLE:
+ ss << std::to_string(get_double());
+ break;
+ case TYPE_CHAR:
+ case TYPE_VARCHAR:
+ ss << get_string();
+ break;
+ case TYPE_DATE:
+ case TYPE_DATETIME:
+ ss << get_date_string();
+ break;
+ case TYPE_BOOLEAN:
+ ss << std::to_string(get_bool());
+ break;
+ case TYPE_DECIMAL:
+ ss << get_decimal_string();
+ break;
+ case TYPE_DECIMALV2:
+ ss << get_decimalv2_string();
+ break;
+ case TYPE_LARGEINT:
+ ss << get_largeint_string();
+ break;
+ default:
+ DCHECK(false);
+ break;
}
return ss.str();
}
-ExtLiteral::~ExtLiteral(){
-}
+ExtLiteral::~ExtLiteral() {}
int8_t ExtLiteral::get_byte() {
DCHECK(_type == TYPE_TINYINT);
@@ -152,7 +150,7 @@ std::string ExtLiteral::get_date_string() {
char str[MAX_DTVALUE_STR_LEN];
date_value.to_string(str);
- return std::string(str, strlen(str));
+ return std::string(str, strlen(str));
}
bool ExtLiteral::get_bool() {
@@ -175,17 +173,15 @@ std::string ExtLiteral::get_largeint_string() {
return LargeIntValue::to_string(*reinterpret_cast<__int128*>(_value));
}
-EsPredicate::EsPredicate(ExprContext* context,
- const TupleDescriptor* tuple_desc, ObjectPool* pool) :
- _context(context),
- _disjuncts_num(0),
- _tuple_desc(tuple_desc),
- _es_query_status(Status::OK()),
- _pool(pool) {
-}
+EsPredicate::EsPredicate(ExprContext* context, const TupleDescriptor* tuple_desc, ObjectPool* pool)
+ : _context(context),
+ _disjuncts_num(0),
+ _tuple_desc(tuple_desc),
+ _es_query_status(Status::OK()),
+ _pool(pool) {}
EsPredicate::~EsPredicate() {
- for(int i=0; i < _disjuncts.size(); i++) {
+ for (int i = 0; i < _disjuncts.size(); i++) {
delete _disjuncts[i];
}
_disjuncts.clear();
@@ -196,7 +192,7 @@ Status EsPredicate::build_disjuncts_list() {
}
// make sure to build by build_disjuncts_list
-const std::vector<ExtPredicate*>& EsPredicate::get_predicate_list(){
+const std::vector<ExtPredicate*>& EsPredicate::get_predicate_list() {
return _disjuncts;
}
@@ -212,16 +208,16 @@ static bool ignore_cast(const SlotDescriptor* slot, const Expr* expr) {
static bool is_literal_node(const Expr* expr) {
switch (expr->node_type()) {
- case TExprNodeType::BOOL_LITERAL:
- case TExprNodeType::INT_LITERAL:
- case TExprNodeType::LARGE_INT_LITERAL:
- case TExprNodeType::FLOAT_LITERAL:
- case TExprNodeType::DECIMAL_LITERAL:
- case TExprNodeType::STRING_LITERAL:
- case TExprNodeType::DATE_LITERAL:
- return true;
- default:
- return false;
+ case TExprNodeType::BOOL_LITERAL:
+ case TExprNodeType::INT_LITERAL:
+ case TExprNodeType::LARGE_INT_LITERAL:
+ case TExprNodeType::FLOAT_LITERAL:
+ case TExprNodeType::DECIMAL_LITERAL:
+ case TExprNodeType::STRING_LITERAL:
+ case TExprNodeType::DATE_LITERAL:
+ return true;
+ default:
+ return false;
}
}
@@ -240,8 +236,8 @@ Status EsPredicate::build_disjuncts_list(const Expr* conjunct) {
// conjunct->get_child(1)->node_type()return FLOAT_LITERAL
// the left child is literal and right child is SlotRef maybe not happened, but here we just process
// this situation regardless of the rewrite logic from the FE's Query Engine
- if (TExprNodeType::SLOT_REF == conjunct->get_child(0)->node_type()
- || TExprNodeType::CAST_EXPR == conjunct->get_child(0)->node_type()) {
+ if (TExprNodeType::SLOT_REF == conjunct->get_child(0)->node_type() ||
+ TExprNodeType::CAST_EXPR == conjunct->get_child(0)->node_type()) {
expr = conjunct->get_child(1);
// process such as sub-query: select * from (select split_part(k, "_", 1) as new_field from table) t where t.new_field > 1;
RETURN_ERROR_IF_EXPR_IS_NOT_SLOTREF(conjunct->get_child(0));
@@ -249,8 +245,8 @@ Status EsPredicate::build_disjuncts_list(const Expr* conjunct) {
// k (float) > 2.0, k(int) > 3.2
slot_ref = (SlotRef*)Expr::expr_without_cast(conjunct->get_child(0));
op = conjunct->op();
- } else if (TExprNodeType::SLOT_REF == conjunct->get_child(1)->node_type()
- || TExprNodeType::CAST_EXPR == conjunct->get_child(1)->node_type()) {
+ } else if (TExprNodeType::SLOT_REF == conjunct->get_child(1)->node_type() ||
+ TExprNodeType::CAST_EXPR == conjunct->get_child(1)->node_type()) {
expr = conjunct->get_child(0);
RETURN_ERROR_IF_EXPR_IS_NOT_SLOTREF(conjunct->get_child(1));
slot_ref = (SlotRef*)Expr::expr_without_cast(conjunct->get_child(1));
@@ -273,12 +269,8 @@ Status EsPredicate::build_disjuncts_list(const Expr* conjunct) {
if (_field_context.find(col) != _field_context.end()) {
col = _field_context[col];
}
- ExtPredicate* predicate = new ExtBinaryPredicate(
- TExprNodeType::BINARY_PRED,
- col,
- slot_desc->type(),
- op,
- literal);
+ ExtPredicate* predicate = new ExtBinaryPredicate(TExprNodeType::BINARY_PRED, col,
+ slot_desc->type(), op, literal);
_disjuncts.push_back(predicate);
return Status::OK();
@@ -295,13 +287,10 @@ Status EsPredicate::build_disjuncts_list(const Expr* conjunct) {
std::vector<ExtLiteral> query_conditions;
query_conditions.emplace_back(literal);
std::vector<ExtColumnDesc> cols;
- ExtPredicate* predicate = new ExtFunction(
- TExprNodeType::FUNCTION_CALL,
- "esquery",
- cols,
- query_conditions);
+ ExtPredicate* predicate = new ExtFunction(TExprNodeType::FUNCTION_CALL, "esquery", cols,
+ query_conditions);
if (_es_query_status.ok()) {
- _es_query_status = BooleanQueryBuilder::check_es_query(*(ExtFunction *)predicate);
+ _es_query_status = BooleanQueryBuilder::check_es_query(*(ExtFunction*)predicate);
if (!_es_query_status.ok()) {
delete predicate;
return _es_query_status;
@@ -326,7 +315,8 @@ Status EsPredicate::build_disjuncts_list(const Expr* conjunct) {
col = _field_context[col];
}
// use TExprNodeType::IS_NULL_PRED for BooleanQueryBuilder translate
- ExtIsNullPredicate* predicate = new ExtIsNullPredicate(TExprNodeType::IS_NULL_PRED, col, slot_desc->type(), is_not_null);
+ ExtIsNullPredicate* predicate = new ExtIsNullPredicate(TExprNodeType::IS_NULL_PRED, col,
+ slot_desc->type(), is_not_null);
_disjuncts.push_back(predicate);
} else if (fname == "like") {
if (conjunct->children().size() != 2) {
@@ -357,30 +347,26 @@ Status EsPredicate::build_disjuncts_list(const Expr* conjunct) {
col = _field_context[col];
}
ExtLiteral literal(type, _context->get_value(expr, NULL));
- ExtPredicate* predicate = new ExtLikePredicate(
- TExprNodeType::LIKE_PRED,
- col,
- slot_desc->type(),
- literal);
+ ExtPredicate* predicate =
+ new ExtLikePredicate(TExprNodeType::LIKE_PRED, col, slot_desc->type(), literal);
_disjuncts.push_back(predicate);
} else {
std::stringstream ss;
- ss << "can not process function predicate[ "
- << fname
- << " ]";
+ ss << "can not process function predicate[ " << fname << " ]";
return Status::InternalError(ss.str());
}
return Status::OK();
}
-
+
if (TExprNodeType::IN_PRED == conjunct->node_type()) {
// the op code maybe FILTER_NEW_IN, it means there is function in list
// like col_a in (abs(1))
- if (TExprOpcode::FILTER_IN != conjunct->op()
- && TExprOpcode::FILTER_NOT_IN != conjunct->op()) {
- return Status::InternalError("build disjuncts failed: "
- "opcode in IN_PRED is neither FILTER_IN nor FILTER_NOT_IN");
+ if (TExprOpcode::FILTER_IN != conjunct->op() &&
+ TExprOpcode::FILTER_NOT_IN != conjunct->op()) {
+ return Status::InternalError(
+ "build disjuncts failed: "
+ "opcode in IN_PRED is neither FILTER_IN nor FILTER_NOT_IN");
}
std::vector<ExtLiteral> in_pred_values;
@@ -390,7 +376,7 @@ Status EsPredicate::build_disjuncts_list(const Expr* conjunct) {
return Status::InternalError("build disjuncts failed: node type is not slot ref");
}
- const SlotDescriptor* slot_desc = get_slot_desc((const SlotRef *)expr);
+ const SlotDescriptor* slot_desc = get_slot_desc((const SlotRef*)expr);
if (slot_desc == nullptr) {
return Status::InternalError("build disjuncts failed: slot_desc is null");
}
@@ -407,7 +393,7 @@ Status EsPredicate::build_disjuncts_list(const Expr* conjunct) {
return Status::InternalError("build disjuncts failed: hybrid set has a null value");
}
- ExtLiteral literal(slot_desc->type().type, const_cast<void *>(iter->get_value()));
+ ExtLiteral literal(slot_desc->type().type, const_cast<void*>(iter->get_value()));
in_pred_values.emplace_back(literal);
iter->next();
}
@@ -415,12 +401,8 @@ Status EsPredicate::build_disjuncts_list(const Expr* conjunct) {
if (_field_context.find(col) != _field_context.end()) {
col = _field_context[col];
}
- ExtPredicate* predicate = new ExtInPredicate(
- TExprNodeType::IN_PRED,
- pred->is_not_in(),
- col,
- slot_desc->type(),
- in_pred_values);
+ ExtPredicate* predicate = new ExtInPredicate(TExprNodeType::IN_PRED, pred->is_not_in(), col,
+ slot_desc->type(), in_pred_values);
_disjuncts.push_back(predicate);
return Status::OK();
@@ -432,7 +414,7 @@ Status EsPredicate::build_disjuncts_list(const Expr* conjunct) {
if (conjunct->op() == TExprOpcode::COMPOUND_AND) {
std::vector<EsPredicate*> conjuncts;
for (int i = 0; i < conjunct->get_num_children(); ++i) {
- EsPredicate *predicate = _pool->add(new EsPredicate(_context, _tuple_desc, _pool));
+ EsPredicate* predicate = _pool->add(new EsPredicate(_context, _tuple_desc, _pool));
predicate->set_field_context(_field_context);
Status status = predicate->build_disjuncts_list(conjunct->children()[i]);
if (status.ok()) {
@@ -441,10 +423,11 @@ Status EsPredicate::build_disjuncts_list(const Expr* conjunct) {
return Status::InternalError("build COMPOUND_AND conjuncts failed");
}
}
- ExtCompPredicates *compound_predicate = new ExtCompPredicates(TExprOpcode::COMPOUND_AND, conjuncts);
+ ExtCompPredicates* compound_predicate =
+ new ExtCompPredicates(TExprOpcode::COMPOUND_AND, conjuncts);
_disjuncts.push_back(compound_predicate);
return Status::OK();
- } else if (conjunct->op() == TExprOpcode::COMPOUND_NOT){
+ } else if (conjunct->op() == TExprOpcode::COMPOUND_NOT) {
// reserved for processing COMPOUND_NOT
return Status::InternalError("currently do not support COMPOUND_NOT push-down");
}
@@ -477,4 +460,4 @@ const SlotDescriptor* EsPredicate::get_slot_desc(const SlotRef* slotRef) {
return slot_desc;
}
-}
+} // namespace doris
diff --git a/be/src/exec/es/es_predicate.h b/be/src/exec/es/es_predicate.h
index a3c6c4f..0ffbe54 100644
--- a/be/src/exec/es/es_predicate.h
+++ b/be/src/exec/es/es_predicate.h
@@ -15,8 +15,8 @@
// specific language governing permissions and limitations
// under the License.
-#ifndef BE_EXEC_ES_PREDICATE_H
-#define BE_EXEC_ES_PREDICATE_H
+#ifndef BE_EXEC_ES_PREDICATE_H
+#define BE_EXEC_ES_PREDICATE_H
#include <string>
#include <vector>
@@ -26,8 +26,8 @@
#include "gen_cpp/Opcodes_types.h"
#include "gen_cpp/PaloExternalDataSourceService_types.h"
#include "runtime/descriptors.h"
-#include "runtime/tuple.h"
#include "runtime/primitive_type.h"
+#include "runtime/tuple.h"
namespace doris {
@@ -38,15 +38,11 @@ class EsPredicate;
class ExtLiteral {
public:
- ExtLiteral(PrimitiveType type, void *value) :
- _type(type),
- _value(value) {
- _str = value_to_string();
+ ExtLiteral(PrimitiveType type, void* value) : _type(type), _value(value) {
+ _str = value_to_string();
}
~ExtLiteral();
- const std::string& to_string() const {
- return _str;
- }
+ const std::string& to_string() const { return _str; }
private:
int8_t get_byte();
@@ -70,18 +66,14 @@ private:
};
struct ExtColumnDesc {
- ExtColumnDesc(const std::string& name, const TypeDescriptor& type) :
- name(name),
- type(type) {
- }
+ ExtColumnDesc(const std::string& name, const TypeDescriptor& type) : name(name), type(type) {}
std::string name;
TypeDescriptor type;
};
struct ExtPredicate {
- ExtPredicate(TExprNodeType::type node_type) : node_type(node_type) {
- }
+ ExtPredicate(TExprNodeType::type node_type) : node_type(node_type) {}
virtual ~ExtPredicate() {}
TExprNodeType::type node_type;
@@ -90,30 +82,17 @@ struct ExtPredicate {
// this used for placeholder for compound_predicate
// reserved for compound_not
struct ExtCompPredicates : public ExtPredicate {
- ExtCompPredicates(
- TExprOpcode::type expr_op,
- const std::vector<EsPredicate*>& es_predicates) :
- ExtPredicate(TExprNodeType::COMPOUND_PRED),
- op(expr_op),
- conjuncts(es_predicates) {
- }
+ ExtCompPredicates(TExprOpcode::type expr_op, const std::vector<EsPredicate*>& es_predicates)
+ : ExtPredicate(TExprNodeType::COMPOUND_PRED), op(expr_op), conjuncts(es_predicates) {}
TExprOpcode::type op;
std::vector<EsPredicate*> conjuncts;
};
struct ExtBinaryPredicate : public ExtPredicate {
- ExtBinaryPredicate(
- TExprNodeType::type node_type,
- const std::string& name,
- const TypeDescriptor& type,
- TExprOpcode::type op,
- const ExtLiteral& value) :
- ExtPredicate(node_type),
- col(name, type),
- op(op),
- value(value) {
- }
+ ExtBinaryPredicate(TExprNodeType::type node_type, const std::string& name,
+ const TypeDescriptor& type, TExprOpcode::type op, const ExtLiteral& value)
+ : ExtPredicate(node_type), col(name, type), op(op), value(value) {}
ExtColumnDesc col;
TExprOpcode::type op;
@@ -121,17 +100,9 @@ struct ExtBinaryPredicate : public ExtPredicate {
};
struct ExtInPredicate : public ExtPredicate {
- ExtInPredicate(
- TExprNodeType::type node_type,
- bool is_not_in,
- const std::string& name,
- const TypeDescriptor& type,
- const std::vector<ExtLiteral>& values) :
- ExtPredicate(node_type),
- is_not_in(is_not_in),
- col(name, type),
- values(values) {
- }
+ ExtInPredicate(TExprNodeType::type node_type, bool is_not_in, const std::string& name,
+ const TypeDescriptor& type, const std::vector<ExtLiteral>& values)
+ : ExtPredicate(node_type), is_not_in(is_not_in), col(name, type), values(values) {}
bool is_not_in;
ExtColumnDesc col;
@@ -139,45 +110,27 @@ struct ExtInPredicate : public ExtPredicate {
};
struct ExtLikePredicate : public ExtPredicate {
- ExtLikePredicate(
- TExprNodeType::type node_type,
- const std::string& name,
- const TypeDescriptor& type,
- ExtLiteral value) :
- ExtPredicate(node_type),
- col(name, type),
- value(value) {
- }
+ ExtLikePredicate(TExprNodeType::type node_type, const std::string& name,
+ const TypeDescriptor& type, ExtLiteral value)
+ : ExtPredicate(node_type), col(name, type), value(value) {}
ExtColumnDesc col;
ExtLiteral value;
};
struct ExtIsNullPredicate : public ExtPredicate {
- ExtIsNullPredicate(
- TExprNodeType::type node_type,
- const std::string& name,
- const TypeDescriptor& type,
- bool is_not_null) :
- ExtPredicate(node_type),
- col(name, type),
- is_not_null(is_not_null) {
- }
+ ExtIsNullPredicate(TExprNodeType::type node_type, const std::string& name,
+ const TypeDescriptor& type, bool is_not_null)
+ : ExtPredicate(node_type), col(name, type), is_not_null(is_not_null) {}
ExtColumnDesc col;
bool is_not_null;
};
struct ExtFunction : public ExtPredicate {
- ExtFunction(TExprNodeType::type node_type,
- const std::string& func_name,
- std::vector<ExtColumnDesc> cols,
- std::vector<ExtLiteral> values) :
- ExtPredicate(node_type),
- func_name(func_name),
- cols(cols),
- values(values) {
- }
+ ExtFunction(TExprNodeType::type node_type, const std::string& func_name,
+ std::vector<ExtColumnDesc> cols, std::vector<ExtLiteral> values)
+ : ExtPredicate(node_type), func_name(func_name), cols(cols), values(values) {}
const std::string func_name;
std::vector<ExtColumnDesc> cols;
@@ -191,13 +144,9 @@ public:
const std::vector<ExtPredicate*>& get_predicate_list();
Status build_disjuncts_list();
// public for tests
- EsPredicate(const std::vector<ExtPredicate*>& all_predicates) {
- _disjuncts = all_predicates;
- };
+ EsPredicate(const std::vector<ExtPredicate*>& all_predicates) { _disjuncts = all_predicates; };
- Status get_es_query_status() {
- return _es_query_status;
- }
+ Status get_es_query_status() { return _es_query_status; }
void set_field_context(const std::map<std::string, std::string>& field_context) {
_field_context = field_context;
@@ -207,15 +156,15 @@ private:
Status build_disjuncts_list(const Expr* conjunct);
const SlotDescriptor* get_slot_desc(const SlotRef* slotRef);
- ExprContext* _context;
+ ExprContext* _context;
int _disjuncts_num;
const TupleDescriptor* _tuple_desc;
std::vector<ExtPredicate*> _disjuncts;
Status _es_query_status;
- ObjectPool *_pool;
+ ObjectPool* _pool;
std::map<std::string, std::string> _field_context;
};
-}
+} // namespace doris
#endif
diff --git a/be/src/exec/es/es_query_builder.cpp b/be/src/exec/es/es_query_builder.cpp
index 441aa55..a75ed0e 100644
--- a/be/src/exec/es/es_query_builder.cpp
+++ b/be/src/exec/es/es_query_builder.cpp
@@ -18,16 +18,15 @@
#include "exec/es/es_query_builder.h"
#include <boost/algorithm/string/replace.hpp>
+
+#include "common/logging.h"
#include "rapidjson/rapidjson.h"
#include "rapidjson/stringbuffer.h"
#include "rapidjson/writer.h"
-#include "common/logging.h"
namespace doris {
-ESQueryBuilder::ESQueryBuilder(const std::string& es_query_str) : _es_query_str(es_query_str) {
-
-}
+ESQueryBuilder::ESQueryBuilder(const std::string& es_query_str) : _es_query_str(es_query_str) {}
... 216555 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org