You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by pa...@apache.org on 2023/11/06 09:32:37 UTC
(doris) branch new_join updated: fix profile and change timer profile (#26468)
This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch new_join
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/new_join by this push:
new 20b3070b17d fix profile and change timer profile (#26468)
20b3070b17d is described below
commit 20b3070b17de5c1f717c1bda6f455806fbdc5a83
Author: HappenLee <ha...@hotmail.com>
AuthorDate: Mon Nov 6 17:32:28 2023 +0800
fix profile and change timer profile (#26468)
---
be/src/pipeline/exec/hashjoin_build_sink.cpp | 7 -------
be/src/pipeline/exec/hashjoin_build_sink.h | 8 --------
be/src/pipeline/exec/join_build_sink_operator.cpp | 8 ++++----
be/src/pipeline/exec/join_build_sink_operator.h | 4 ++--
.../exec/nested_loop_join_build_operator.h | 8 ++++++--
be/src/vec/common/hash_table/hash_map.h | 5 +++++
be/src/vec/exec/join/vhash_join_node.cpp | 8 --------
be/src/vec/exec/join/vhash_join_node.h | 22 +++++++---------------
be/src/vec/exec/join/vjoin_node_base.cpp | 4 ++--
be/src/vec/exec/join/vjoin_node_base.h | 4 ++--
be/src/vec/exec/join/vnested_loop_join_node.cpp | 4 ++--
be/src/vec/exec/join/vnested_loop_join_node.h | 8 ++++++--
12 files changed, 36 insertions(+), 54 deletions(-)
diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp
index 0cdb34605e0..c7af4f89ba4 100644
--- a/be/src/pipeline/exec/hashjoin_build_sink.cpp
+++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp
@@ -92,17 +92,10 @@ Status HashJoinBuildSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo
_build_side_merge_block_timer = ADD_TIMER(profile(), "BuildSideMergeBlockTime");
_build_table_insert_timer = ADD_TIMER(record_profile, "BuildTableInsertTime");
_build_expr_call_timer = ADD_TIMER(record_profile, "BuildExprCallTime");
- _build_table_expanse_timer = ADD_TIMER(record_profile, "BuildTableExpanseTime");
- _build_table_convert_timer = ADD_TIMER(record_profile, "BuildTableConvertToPartitionedTime");
_build_side_compute_hash_timer = ADD_TIMER(record_profile, "BuildSideHashComputingTime");
- _build_runtime_filter_timer = ADD_TIMER(record_profile, "BuildRuntimeFilterTime");
_allocate_resource_timer = ADD_TIMER(profile(), "AllocateResourceTime");
- _build_buckets_counter = ADD_COUNTER(profile(), "BuildBuckets", TUnit::UNIT);
- _build_buckets_fill_counter = ADD_COUNTER(profile(), "FilledBuckets", TUnit::UNIT);
-
- _build_collisions_counter = ADD_COUNTER(profile(), "BuildCollisions", TUnit::UNIT);
// Hash Table Init
_hash_table_init(state);
diff --git a/be/src/pipeline/exec/hashjoin_build_sink.h b/be/src/pipeline/exec/hashjoin_build_sink.h
index 16b58adf5fc..dc047e39848 100644
--- a/be/src/pipeline/exec/hashjoin_build_sink.h
+++ b/be/src/pipeline/exec/hashjoin_build_sink.h
@@ -107,16 +107,8 @@ protected:
RuntimeProfile::Counter* _build_table_timer;
RuntimeProfile::Counter* _build_expr_call_timer;
RuntimeProfile::Counter* _build_table_insert_timer;
- RuntimeProfile::Counter* _build_table_expanse_timer;
- RuntimeProfile::Counter* _build_table_convert_timer;
- RuntimeProfile::Counter* _build_buckets_counter;
- RuntimeProfile::Counter* _build_buckets_fill_counter;
-
RuntimeProfile::Counter* _build_side_compute_hash_timer;
RuntimeProfile::Counter* _build_side_merge_block_timer;
- RuntimeProfile::Counter* _build_runtime_filter_timer;
-
- RuntimeProfile::Counter* _build_collisions_counter;
RuntimeProfile::Counter* _allocate_resource_timer;
diff --git a/be/src/pipeline/exec/join_build_sink_operator.cpp b/be/src/pipeline/exec/join_build_sink_operator.cpp
index 2ed4ebfb0d3..280a35a0049 100644
--- a/be/src/pipeline/exec/join_build_sink_operator.cpp
+++ b/be/src/pipeline/exec/join_build_sink_operator.cpp
@@ -35,10 +35,10 @@ Status JoinBuildSinkLocalState<DependencyType, Derived>::init(RuntimeState* stat
_build_rows_counter = ADD_COUNTER(PipelineXSinkLocalState<DependencyType>::profile(),
"BuildRows", TUnit::UNIT);
- _push_down_timer = ADD_TIMER(PipelineXSinkLocalState<DependencyType>::profile(),
- "PublishRuntimeFilterTime");
- _push_compute_timer =
- ADD_TIMER(PipelineXSinkLocalState<DependencyType>::profile(), "PushDownComputeTime");
+ _publish_runtime_filter_timer = ADD_TIMER(PipelineXSinkLocalState<DependencyType>::profile(),
+ "PublishRuntimeFilterTime");
+ _runtime_filter_compute_timer = ADD_TIMER(PipelineXSinkLocalState<DependencyType>::profile(),
+ "RuntimeFilterComputeTime");
return Status::OK();
}
diff --git a/be/src/pipeline/exec/join_build_sink_operator.h b/be/src/pipeline/exec/join_build_sink_operator.h
index 9034057658a..369b1548cfd 100644
--- a/be/src/pipeline/exec/join_build_sink_operator.h
+++ b/be/src/pipeline/exec/join_build_sink_operator.h
@@ -41,8 +41,8 @@ protected:
friend class JoinBuildSinkOperatorX;
RuntimeProfile::Counter* _build_rows_counter;
- RuntimeProfile::Counter* _push_down_timer;
- RuntimeProfile::Counter* _push_compute_timer;
+ RuntimeProfile::Counter* _publish_runtime_filter_timer;
+ RuntimeProfile::Counter* _runtime_filter_compute_timer;
};
template <typename LocalStateType>
diff --git a/be/src/pipeline/exec/nested_loop_join_build_operator.h b/be/src/pipeline/exec/nested_loop_join_build_operator.h
index bd94c9e0e9d..28375d93d09 100644
--- a/be/src/pipeline/exec/nested_loop_join_build_operator.h
+++ b/be/src/pipeline/exec/nested_loop_join_build_operator.h
@@ -59,9 +59,13 @@ public:
const std::vector<TRuntimeFilterDesc>& runtime_filter_descs();
vectorized::VExprContextSPtrs& filter_src_expr_ctxs() { return _filter_src_expr_ctxs; }
- RuntimeProfile::Counter* push_compute_timer() { return _push_compute_timer; }
+ RuntimeProfile::Counter* runtime_filter_compute_timer() {
+ return _runtime_filter_compute_timer;
+ }
vectorized::Blocks& build_blocks() { return _shared_state->build_blocks; }
- RuntimeProfile::Counter* push_down_timer() { return _push_down_timer; }
+ RuntimeProfile::Counter* publish_runtime_filter_timer() {
+ return _publish_runtime_filter_timer;
+ }
private:
friend class NestedLoopJoinBuildSinkOperatorX;
diff --git a/be/src/vec/common/hash_table/hash_map.h b/be/src/vec/common/hash_table/hash_map.h
index a0d02ee0089..a9506869875 100644
--- a/be/src/vec/common/hash_table/hash_map.h
+++ b/be/src/vec/common/hash_table/hash_map.h
@@ -217,6 +217,11 @@ public:
return phmap::priv::NormalizeCapacity(expect_bucket_size) + 1;
}
+ size_t get_byte_size() const {
+ auto cal_vector_mem = [](const auto& vec) { return vec.capacity() * sizeof(vec[0]); };
+ return cal_vector_mem(visited) + cal_vector_mem(first) + cal_vector_mem(next);
+ }
+
template <int JoinOpType>
void prepare_build(size_t num_elem, int batch_size) {
max_batch_size = batch_size;
diff --git a/be/src/vec/exec/join/vhash_join_node.cpp b/be/src/vec/exec/join/vhash_join_node.cpp
index 9e8c99c8d56..4d0ed465400 100644
--- a/be/src/vec/exec/join/vhash_join_node.cpp
+++ b/be/src/vec/exec/join/vhash_join_node.cpp
@@ -225,10 +225,7 @@ Status HashJoinNode::prepare(RuntimeState* state) {
ADD_CHILD_TIMER(record_profile, "BuildSideMergeBlockTime", "BuildTime");
_build_table_insert_timer = ADD_TIMER(record_profile, "BuildTableInsertTime");
_build_expr_call_timer = ADD_TIMER(record_profile, "BuildExprCallTime");
- _build_table_expanse_timer = ADD_TIMER(record_profile, "BuildTableExpanseTime");
- _build_table_convert_timer = ADD_TIMER(record_profile, "BuildTableConvertToPartitionedTime");
_build_side_compute_hash_timer = ADD_TIMER(record_profile, "BuildSideHashComputingTime");
- _build_runtime_filter_timer = ADD_TIMER(record_profile, "BuildRuntimeFilterTime");
// Probe phase
auto probe_phase_profile = _probe_phase_profile;
@@ -246,11 +243,6 @@ Status HashJoinNode::prepare(RuntimeState* state) {
_allocate_resource_timer = ADD_TIMER(runtime_profile(), "AllocateResourceTime");
_process_other_join_conjunct_timer = ADD_TIMER(runtime_profile(), "OtherJoinConjunctTime");
- _build_buckets_counter = ADD_COUNTER(runtime_profile(), "BuildBuckets", TUnit::UNIT);
- _build_buckets_fill_counter = ADD_COUNTER(runtime_profile(), "FilledBuckets", TUnit::UNIT);
-
- _build_collisions_counter = ADD_COUNTER(runtime_profile(), "BuildCollisions", TUnit::UNIT);
-
RETURN_IF_ERROR(VExpr::prepare(_build_expr_ctxs, state, child(1)->row_desc()));
RETURN_IF_ERROR(VExpr::prepare(_probe_expr_ctxs, state, child(0)->row_desc()));
diff --git a/be/src/vec/exec/join/vhash_join_node.h b/be/src/vec/exec/join/vhash_join_node.h
index 7988cc598bd..6d47528fefd 100644
--- a/be/src/vec/exec/join/vhash_join_node.h
+++ b/be/src/vec/exec/join/vhash_join_node.h
@@ -88,12 +88,12 @@ struct ProcessRuntimeFilterBuild {
if (!parent->_runtime_filter_slots->empty() && !parent->_inserted_blocks.empty()) {
{
- SCOPED_TIMER(parent->_push_compute_timer);
+ SCOPED_TIMER(parent->_runtime_filter_compute_timer);
parent->_runtime_filter_slots->insert(parent->_inserted_blocks);
}
}
{
- SCOPED_TIMER(parent->_push_down_timer);
+ SCOPED_TIMER(parent->_publish_runtime_filter_timer);
RETURN_IF_ERROR(parent->_runtime_filter_slots->publish());
}
@@ -112,8 +112,7 @@ struct ProcessHashTableBuild {
_build_raw_ptrs(build_raw_ptrs),
_parent(parent),
_batch_size(batch_size),
- _state(state),
- _build_side_compute_hash_timer(parent->_build_side_compute_hash_timer) {}
+ _state(state) {}
template <int JoinOpType, bool ignore_null, bool short_circuit_for_null>
Status run(HashTableContext& hash_table_ctx, ConstNullMapPtr null_map, bool* has_null_key) {
@@ -133,16 +132,18 @@ struct ProcessHashTableBuild {
}
SCOPED_TIMER(_parent->_build_table_insert_timer);
- hash_table_ctx.hash_table->template prepare_build<JoinOpType>(_rows, _state->batch_size());
+ hash_table_ctx.hash_table->template prepare_build<JoinOpType>(_rows, _batch_size);
hash_table_ctx.init_serialized_keys(_build_raw_ptrs, _rows,
null_map ? null_map->data() : nullptr, true, true,
hash_table_ctx.hash_table->get_bucket_size());
hash_table_ctx.hash_table->build(hash_table_ctx.keys, hash_table_ctx.bucket_nums.data(),
_rows);
- hash_table_ctx.bucket_nums.resize(_state->batch_size());
+ hash_table_ctx.bucket_nums.resize(_batch_size);
hash_table_ctx.bucket_nums.shrink_to_fit();
+ COUNTER_UPDATE(_parent->_hash_table_memory_usage,
+ hash_table_ctx.hash_table->get_byte_size());
return Status::OK();
}
@@ -153,8 +154,6 @@ private:
Parent* _parent;
int _batch_size;
RuntimeState* _state;
-
- ProfileCounter* _build_side_compute_hash_timer;
};
template <typename RowRefListType>
@@ -340,21 +339,14 @@ private:
RuntimeProfile::Counter* _build_table_timer;
RuntimeProfile::Counter* _build_expr_call_timer;
RuntimeProfile::Counter* _build_table_insert_timer;
- RuntimeProfile::Counter* _build_table_expanse_timer;
- RuntimeProfile::Counter* _build_table_convert_timer;
RuntimeProfile::Counter* _probe_expr_call_timer;
RuntimeProfile::Counter* _probe_next_timer;
- RuntimeProfile::Counter* _build_buckets_counter;
- RuntimeProfile::Counter* _build_buckets_fill_counter;
RuntimeProfile::Counter* _search_hashtable_timer;
RuntimeProfile::Counter* _build_side_output_timer;
RuntimeProfile::Counter* _probe_side_output_timer;
RuntimeProfile::Counter* _probe_process_hashtable_timer;
RuntimeProfile::Counter* _build_side_compute_hash_timer;
RuntimeProfile::Counter* _build_side_merge_block_timer;
- RuntimeProfile::Counter* _build_runtime_filter_timer;
-
- RuntimeProfile::Counter* _build_collisions_counter;
RuntimeProfile::Counter* _open_timer;
RuntimeProfile::Counter* _allocate_resource_timer;
diff --git a/be/src/vec/exec/join/vjoin_node_base.cpp b/be/src/vec/exec/join/vjoin_node_base.cpp
index 4cdb7aed78c..2b9cb714e1a 100644
--- a/be/src/vec/exec/join/vjoin_node_base.cpp
+++ b/be/src/vec/exec/join/vjoin_node_base.cpp
@@ -123,8 +123,8 @@ Status VJoinNodeBase::prepare(RuntimeState* state) {
ADD_CHILD_TIMER(_probe_phase_profile, "BuildOutputBlock", "ProbeTime");
_probe_rows_counter = ADD_COUNTER_WITH_LEVEL(_probe_phase_profile, "ProbeRows", TUnit::UNIT, 1);
- _push_down_timer = ADD_TIMER(runtime_profile(), "PublishRuntimeFilterTime");
- _push_compute_timer = ADD_TIMER(runtime_profile(), "PushDownComputeTime");
+ _publish_runtime_filter_timer = ADD_TIMER(runtime_profile(), "PublishRuntimeFilterTime");
+ _runtime_filter_compute_timer = ADD_TIMER(runtime_profile(), "RunmtimeFilterComputeTime");
return Status::OK();
}
diff --git a/be/src/vec/exec/join/vjoin_node_base.h b/be/src/vec/exec/join/vjoin_node_base.h
index ad68faa5c0a..2fdc11d2053 100644
--- a/be/src/vec/exec/join/vjoin_node_base.h
+++ b/be/src/vec/exec/join/vjoin_node_base.h
@@ -142,8 +142,8 @@ protected:
RuntimeProfile* _probe_phase_profile;
RuntimeProfile::Counter* _probe_timer;
RuntimeProfile::Counter* _probe_rows_counter;
- RuntimeProfile::Counter* _push_down_timer;
- RuntimeProfile::Counter* _push_compute_timer;
+ RuntimeProfile::Counter* _publish_runtime_filter_timer;
+ RuntimeProfile::Counter* _runtime_filter_compute_timer;
RuntimeProfile::Counter* _join_filter_timer;
RuntimeProfile::Counter* _build_output_block_timer;
};
diff --git a/be/src/vec/exec/join/vnested_loop_join_node.cpp b/be/src/vec/exec/join/vnested_loop_join_node.cpp
index b4f5814a3af..e7407b3e2d2 100644
--- a/be/src/vec/exec/join/vnested_loop_join_node.cpp
+++ b/be/src/vec/exec/join/vnested_loop_join_node.cpp
@@ -78,13 +78,13 @@ Status RuntimeFilterBuild<Parent>::operator()(RuntimeState* state) {
RETURN_IF_ERROR(runtime_filter_slots.init(state));
if (!runtime_filter_slots.empty() && !_parent->build_blocks().empty()) {
- SCOPED_TIMER(_parent->push_compute_timer());
+ SCOPED_TIMER(_parent->runtime_filter_compute_timer());
for (auto& build_block : _parent->build_blocks()) {
RETURN_IF_ERROR(runtime_filter_slots.insert(&build_block));
}
}
{
- SCOPED_TIMER(_parent->push_down_timer());
+ SCOPED_TIMER(_parent->publish_runtime_filter_timer());
RETURN_IF_ERROR(runtime_filter_slots.publish());
}
diff --git a/be/src/vec/exec/join/vnested_loop_join_node.h b/be/src/vec/exec/join/vnested_loop_join_node.h
index bc5944eefa2..b0761ea79a5 100644
--- a/be/src/vec/exec/join/vnested_loop_join_node.h
+++ b/be/src/vec/exec/join/vnested_loop_join_node.h
@@ -99,9 +99,13 @@ public:
std::vector<TRuntimeFilterDesc>& runtime_filter_descs() { return _runtime_filter_descs; }
VExprContextSPtrs& filter_src_expr_ctxs() { return _filter_src_expr_ctxs; }
- RuntimeProfile::Counter* push_compute_timer() { return _push_compute_timer; }
+ RuntimeProfile::Counter* runtime_filter_compute_timer() {
+ return _runtime_filter_compute_timer;
+ }
Blocks& build_blocks() { return _build_blocks; }
- RuntimeProfile::Counter* push_down_timer() { return _push_down_timer; }
+ RuntimeProfile::Counter* publish_runtime_filter_timer() {
+ return _publish_runtime_filter_timer;
+ }
private:
template <typename JoinOpType, bool set_build_side_flag, bool set_probe_side_flag>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org