You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/07/18 15:39:03 UTC
(doris) branch branch-2.1 updated: [cherrypick] Fix profile eject (#38095)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 426493a7f18 [cherrypick] Fix profile eject (#38095)
426493a7f18 is described below
commit 426493a7f18c0e4439aa01efee8bf57f03e1079d
Author: zhiqiang <se...@163.com>
AuthorDate: Thu Jul 18 23:38:56 2024 +0800
[cherrypick] Fix profile eject (#38095)
pick https://github.com/apache/doris/pull/38030
---
.../apache/doris/common/util/ProfileManager.java | 28 ++++++++++++++++------
.../java/org/apache/doris/qe/QeProcessorImpl.java | 5 ++--
2 files changed, 24 insertions(+), 9 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java
index 70bb21a27e9..50543de05b1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java
@@ -153,22 +153,27 @@ public class ProfileManager {
LOG.debug("Add execution profile {} to profile manager",
DebugUtil.printId(executionProfile.getQueryId()));
}
- // Check if there are some query profiles that not finish collecting, should
- // remove them to release memory.
+ // This branch has two purposes:
+ // 1. discard profile collecting if its collection not finished in 5 seconds after query finished.
+ // 2. prevent execution profile from leakage. If we have too many execution profiles in memory,
+ // we will remove execution profiles of query that has finished in 5 seconds ago.
if (queryIdToExecutionProfiles.size() > 2 * Config.max_query_profile_num) {
List<ExecutionProfile> finishOrExpireExecutionProfiles = Lists.newArrayList();
for (ExecutionProfile tmpProfile : queryIdToExecutionProfiles.values()) {
- if (System.currentTimeMillis() - tmpProfile.getQueryFinishTime()
- > Config.profile_async_collect_expire_time_secs * 1000) {
+ boolean queryFinishedLongEnough = tmpProfile.getQueryFinishTime() > 0
+ && System.currentTimeMillis() - tmpProfile.getQueryFinishTime()
+ > Config.profile_async_collect_expire_time_secs * 1000;
+
+ if (queryFinishedLongEnough) {
finishOrExpireExecutionProfiles.add(tmpProfile);
}
}
+ StringBuilder stringBuilder = new StringBuilder();
for (ExecutionProfile tmp : finishOrExpireExecutionProfiles) {
+ stringBuilder.append(DebugUtil.printId(tmp.getQueryId())).append(",");
queryIdToExecutionProfiles.remove(tmp.getQueryId());
- if (LOG.isDebugEnabled()) {
- LOG.debug("Remove expired execution profile {}", DebugUtil.printId(tmp.getQueryId()));
- }
}
+ LOG.warn("Remove expired execution profiles {}", stringBuilder.toString());
}
} finally {
writeLock.unlock();
@@ -203,10 +208,19 @@ public class ProfileManager {
ProfileElement profileElementRemoved = queryIdToProfileMap.remove(queryIdDeque.getFirst());
// If the Profile object is removed from manager, then related execution profile is also useless.
if (profileElementRemoved != null) {
+ StringBuilder sb = new StringBuilder();
for (ExecutionProfile executionProfile : profileElementRemoved.profile.getExecutionProfiles()) {
+ sb.append(DebugUtil.printId(executionProfile.getQueryId())).append(",");
this.queryIdToExecutionProfiles.remove(executionProfile.getQueryId());
}
+ LOG.warn("Remove expired profile {}, execution profiles {},"
+ + " queryIdDeque size {}, profile count {},"
+ + " execution profile count {} max_query_profile_num {}",
+ profileElementRemoved.profile.getSummaryProfile().getProfileId(),
+ sb.toString(), queryIdDeque.size(), queryIdToProfileMap.size(),
+ queryIdToExecutionProfiles.size(), Config.max_query_profile_num);
}
+
queryIdDeque.removeFirst();
}
queryIdDeque.addLast(key);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java b/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java
index 8bc189321bd..26bb3d95db3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java
@@ -99,10 +99,11 @@ public final class QeProcessorImpl implements QeProcessor {
if (result != null) {
throw new UserException("queryId " + queryId + " already exists");
}
-
// Should add the execution profile to profile manager, BE will report the profile to FE and FE
// will update it in ProfileManager
- ProfileManager.getInstance().addExecutionProfile(info.getCoord().getExecutionProfile());
+ if (info.coord.getQueryOptions().enable_profile) {
+ ProfileManager.getInstance().addExecutionProfile(info.getCoord().getExecutionProfile());
+ }
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org