You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2020/04/24 13:39:06 UTC
[incubator-doris] branch master updated: [Profile] Make running
profile clearer and more intuitive to improve usability (#3365) (#3383)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4eb27bc [Profile] Make running profile clearer and more intuitive to improve usability (#3365) (#3383)
4eb27bc is described below
commit 4eb27bc7e32222d81b3a6a9b2bededb9af01d8e9
Author: HappenLee <ha...@hotmail.com>
AuthorDate: Fri Apr 24 21:38:55 2020 +0800
[Profile] Make running profile clearer and more intuitive to improve usability (#3365) (#3383)
This CL mainly made the following modifications:
1. Delete Invalid method in Running Profile Class.
2. Move Memlimit Counter from blockmgr to fragment and add PeakMemUsage Counter
3. Fix the bug of buffer pool memlimit counter
4. Call compute_time_in_profile() before pretty_print() to show the _local_time_percent without child running profile
5. Add TransferThread ThreadToken count in AveThreadToken Counter
---
be/src/exec/exec_node.cpp | 2 +-
be/src/exec/olap_scan_node.cpp | 2 ++
be/src/runtime/buffered_block_mgr2.cc | 2 --
be/src/runtime/plan_fragment_executor.cpp | 8 ++++++++
be/src/runtime/runtime_state.cpp | 6 +++++-
be/src/util/runtime_profile.cpp | 3 ++-
be/src/util/runtime_profile.h | 2 --
7 files changed, 18 insertions(+), 7 deletions(-)
diff --git a/be/src/exec/exec_node.cpp b/be/src/exec/exec_node.cpp
index 7bdc3ff..4bde878 100644
--- a/be/src/exec/exec_node.cpp
+++ b/be/src/exec/exec_node.cpp
@@ -596,7 +596,7 @@ Status ExecNode::claim_buffer_reservation(RuntimeState* state) {
ss << print_plan_node_type(_type) << " id=" << _id << " ptr=" << this;
RETURN_IF_ERROR(buffer_pool->RegisterClient(ss.str(),
state->instance_buffer_reservation(),
- mem_tracker(), _resource_profile.max_reservation,
+ mem_tracker(), buffer_pool->GetSystemBytesLimit(),
runtime_profile(),
&_buffer_pool_client));
diff --git a/be/src/exec/olap_scan_node.cpp b/be/src/exec/olap_scan_node.cpp
index 921b539..77b798b 100644
--- a/be/src/exec/olap_scan_node.cpp
+++ b/be/src/exec/olap_scan_node.cpp
@@ -1038,6 +1038,7 @@ Status OlapScanNode::normalize_binary_predicate(SlotDescriptor* slot, ColumnValu
void OlapScanNode::transfer_thread(RuntimeState* state) {
// scanner open pushdown to scanThread
+ state->resource_pool()->acquire_thread_token();
Status status = Status::OK();
for (auto scanner : _olap_scanners) {
status = Expr::clone_if_not_exists(_conjunct_ctxs, state, scanner->conjunct_ctxs());
@@ -1171,6 +1172,7 @@ void OlapScanNode::transfer_thread(RuntimeState* state) {
}
}
+ state->resource_pool()->release_thread_token(true);
VLOG(1) << "TransferThread finish.";
boost::unique_lock<boost::mutex> l(_row_batches_lock);
_transfer_done = true;
diff --git a/be/src/runtime/buffered_block_mgr2.cc b/be/src/runtime/buffered_block_mgr2.cc
index ee38101..2dfad5c 100644
--- a/be/src/runtime/buffered_block_mgr2.cc
+++ b/be/src/runtime/buffered_block_mgr2.cc
@@ -1268,8 +1268,6 @@ void BufferedBlockMgr2::init(
_profile.reset(new RuntimeProfile(&_obj_pool, "BlockMgr"));
parent_profile->add_child(_profile.get(), true, NULL);
- _mem_tracker_counter = ADD_COUNTER(_profile.get(), "MemoryLimit", TUnit::BYTES);
- _mem_tracker_counter->set(mem_limit);
_block_size_counter = ADD_COUNTER(_profile.get(), "MaxBlockSize", TUnit::BYTES);
_block_size_counter->set(_max_block_size);
_created_block_counter = ADD_COUNTER(_profile.get(), "BlocksCreated", TUnit::UNIT);
diff --git a/be/src/runtime/plan_fragment_executor.cpp b/be/src/runtime/plan_fragment_executor.cpp
index 0c7c7fb..f050402 100644
--- a/be/src/runtime/plan_fragment_executor.cpp
+++ b/be/src/runtime/plan_fragment_executor.cpp
@@ -381,6 +381,7 @@ void PlanFragmentExecutor::report_profile() {
VLOG_FILE << "Reporting " << (!_report_thread_active ? "final " : " ")
<< "profile for instance " << _runtime_state->fragment_instance_id();
std::stringstream ss;
+ profile()->compute_time_in_profile();
profile()->pretty_print(&ss);
VLOG_FILE << ss.str();
}
@@ -556,6 +557,13 @@ void PlanFragmentExecutor::close() {
{
std::stringstream ss;
+ // Compute the _local_time_percent before pretty_print the runtime_profile
+ // Before add this operation, the print out like that:
+ // UNION_NODE (id=0):(Active: 56.720us, non-child: 00.00%)
+ // After add thie operation, the print out like that:
+ // UNION_NODE (id=0):(Active: 56.720us, non-child: 82.53%)
+ // We can easily know the exec node excute time without child time consumed.
+ _runtime_state->runtime_profile()->compute_time_in_profile();
_runtime_state->runtime_profile()->pretty_print(&ss);
LOG(INFO) << ss.str();
}
diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp
index 0ee816e..b8e450e 100644
--- a/be/src/runtime/runtime_state.cpp
+++ b/be/src/runtime/runtime_state.cpp
@@ -230,10 +230,14 @@ Status RuntimeState::init_mem_trackers(const TUniqueId& query_id) {
// _query_mem_tracker = MemTracker::get_query_mem_tracker(
// query_id, bytes_limit, _exec_env->process_mem_tracker());
+
+ auto mem_tracker_counter = ADD_COUNTER(&_profile, "MemoryLimit", TUnit::BYTES);
+ mem_tracker_counter->set(bytes_limit);
+
_query_mem_tracker.reset(
new MemTracker(bytes_limit, runtime_profile()->name(), _exec_env->process_mem_tracker()));
_instance_mem_tracker.reset(
- new MemTracker(-1, runtime_profile()->name(), _query_mem_tracker.get()));
+ new MemTracker(&_profile, bytes_limit, runtime_profile()->name(), _query_mem_tracker.get()));
/*
// TODO: this is a stopgap until we implement ExprContext
diff --git a/be/src/util/runtime_profile.cpp b/be/src/util/runtime_profile.cpp
index 51432f7..7b69153 100644
--- a/be/src/util/runtime_profile.cpp
+++ b/be/src/util/runtime_profile.cpp
@@ -47,6 +47,7 @@ static const std::string ROOT_COUNTER = "";
RuntimeProfile::PeriodicCounterUpdateState RuntimeProfile::_s_periodic_counter_update_state;
+// TODO: we do not use the param pool should we del the param ObjectPool
RuntimeProfile::RuntimeProfile(ObjectPool* pool, const std::string& name, bool is_averaged_profile)
: _pool(new ObjectPool()),
_own_pool(false),
@@ -520,7 +521,7 @@ void RuntimeProfile::pretty_print(std::ostream* s, const std::string& prefix) co
if (total_time->second->value() != 0) {
stream << "(Active: "
<< PrettyPrinter::print(total_time->second->value(), total_time->second->type())
- << ", % non-child: " << std::setprecision(2) << _local_time_percent << "%)";
+ << ", non-child: " << std::setprecision(2) << _local_time_percent << "%)";
}
stream << std::endl;
diff --git a/be/src/util/runtime_profile.h b/be/src/util/runtime_profile.h
index d6db81e..1b57776 100644
--- a/be/src/util/runtime_profile.h
+++ b/be/src/util/runtime_profile.h
@@ -106,8 +106,6 @@ public:
_value.store(value);
}
- virtual void set(int value) { _value.store(value); }
-
virtual void set(double value) {
DCHECK_EQ(sizeof(value), sizeof(int64_t));
_value.store(*reinterpret_cast<int64_t*>(&value));
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org