You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by we...@apache.org on 2023/01/15 10:26:23 UTC
[doris] branch master updated: [Enhancement](profile) use queryId of originStatement as the queryId. (#15898)
This is an automated email from the ASF dual-hosted git repository.
weixiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 067b33bca7 [Enhancement](profile) use queryId of originStatement as the queryId. (#15898)
067b33bca7 is described below
commit 067b33bca7c7bf7ad48377f1b36832ad2caa9eb6
Author: wxy <du...@gmail.com>
AuthorDate: Sun Jan 15 18:26:14 2023 +0800
[Enhancement](profile) use queryId of originStatement as the queryId. (#15898)
* [Enhancement](profile) use queryId of originStatement as the queryId.
Co-authored-by: wangxiangyu@360shuke.com <wa...@360shuke.com>
---
.../apache/doris/analysis/ShowQueryProfileStmt.java | 1 +
.../apache/doris/common/util/ProfileManager.java | 21 ++++++++++++++-------
.../main/java/org/apache/doris/load/ExportJob.java | 9 ++++++++-
.../org/apache/doris/load/loadv2/BrokerLoadJob.java | 3 ++-
.../org/apache/doris/load/loadv2/BulkLoadJob.java | 5 +++++
.../main/java/org/apache/doris/qe/StmtExecutor.java | 1 +
.../org/apache/doris/task/ExportExportingTask.java | 3 ++-
7 files changed, 33 insertions(+), 10 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java
index 15be652aed..b36cd30159 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java
@@ -34,6 +34,7 @@ public class ShowQueryProfileStmt extends ShowStmt {
// This should be same as ProfileManager.PROFILE_HEADERS
public static final ShowResultSetMetaData META_DATA_QUERY_IDS =
ShowResultSetMetaData.builder()
+ .addColumn(new Column("JobId", ScalarType.createVarchar(128)))
.addColumn(new Column("QueryId", ScalarType.createVarchar(128)))
.addColumn(new Column("User", ScalarType.createVarchar(128)))
.addColumn(new Column("DefaultDb", ScalarType.createVarchar(128)))
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java
index 6429b0854e..ae945c543e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java
@@ -58,6 +58,8 @@ public class ProfileManager {
private static volatile ProfileManager INSTANCE = null;
// private static final int ARRAY_SIZE = 100;
// private static final int TOTAL_LEN = 1000 * ARRAY_SIZE ;
+ // just use for load profile and export profile
+ public static final String JOB_ID = "Job ID";
public static final String QUERY_ID = "Query ID";
public static final String START_TIME = "Start Time";
public static final String END_TIME = "End Time";
@@ -84,7 +86,7 @@ public class ProfileManager {
}
public static final ArrayList<String> PROFILE_HEADERS = new ArrayList(
- Arrays.asList(QUERY_ID, USER, DEFAULT_DB, SQL_STATEMENT, QUERY_TYPE,
+ Arrays.asList(JOB_ID, QUERY_ID, USER, DEFAULT_DB, SQL_STATEMENT, QUERY_TYPE,
START_TIME, END_TIME, TOTAL_TIME, QUERY_STATE, TRACE_ID));
private class ProfileElement {
@@ -148,25 +150,26 @@ public class ProfileManager {
}
ProfileElement element = createElement(profile);
- String queryId = element.infoStrings.get(ProfileManager.QUERY_ID);
+ String key = isQueryProfile(profile) ? element.infoStrings.get(ProfileManager.QUERY_ID)
+ : element.infoStrings.get(ProfileManager.JOB_ID);
// check when push in, which can ensure every element in the list has QUERY_ID column,
// so there is no need to check when remove element from list.
- if (Strings.isNullOrEmpty(queryId)) {
+ if (Strings.isNullOrEmpty(key)) {
LOG.warn("the key or value of Map is null, "
- + "may be forget to insert 'QUERY_ID' column into infoStrings");
+ + "may be forget to insert 'QUERY_ID' or 'JOB_ID' column into infoStrings");
}
// a profile may be updated multiple times in queryIdToProfileMap,
// and only needs to be inserted into the queryIdDeque for the first time.
- queryIdToProfileMap.put(queryId, element);
+ queryIdToProfileMap.put(key, element);
writeLock.lock();
try {
- if (!queryIdDeque.contains(queryId)) {
+ if (!queryIdDeque.contains(key)) {
if (queryIdDeque.size() >= Config.max_query_profile_num) {
queryIdToProfileMap.remove(queryIdDeque.getFirst());
queryIdDeque.removeFirst();
}
- queryIdDeque.addLast(queryId);
+ queryIdDeque.addLast(key);
}
} finally {
writeLock.unlock();
@@ -333,4 +336,8 @@ public class ProfileManager {
readLock.unlock();
}
}
+
+ public boolean isQueryProfile(RuntimeProfile profile) {
+ return "Query".equals(profile.getName());
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
index 7f681548df..22c931eb3a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
@@ -51,6 +51,7 @@ import org.apache.doris.common.Status;
import org.apache.doris.common.UserException;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
+import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.common.util.SqlParserUtils;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.planner.DataPartition;
@@ -116,6 +117,7 @@ public class ExportJob implements Writable {
}
private long id;
+ private String queryId;
private String label;
private long dbId;
private long tableId;
@@ -175,6 +177,7 @@ public class ExportJob implements Writable {
public ExportJob() {
this.id = -1;
+ this.queryId = "";
this.dbId = -1;
this.tableId = -1;
this.state = JobState.PENDING;
@@ -201,11 +204,11 @@ public class ExportJob implements Writable {
Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbName);
Preconditions.checkNotNull(stmt.getBrokerDesc());
this.brokerDesc = stmt.getBrokerDesc();
-
this.columnSeparator = stmt.getColumnSeparator();
this.lineDelimiter = stmt.getLineDelimiter();
this.properties = stmt.getProperties();
this.label = this.properties.get(ExportStmt.LABEL);
+ this.queryId = ConnectContext.get() != null ? DebugUtil.printId(ConnectContext.get().queryId()) : "N/A";
String path = stmt.getPath();
Preconditions.checkArgument(!Strings.isNullOrEmpty(path));
@@ -729,6 +732,10 @@ public class ExportJob implements Writable {
return label;
}
+ public String getQueryId() {
+ return queryId;
+ }
+
@Override
public String toString() {
return "ExportJob [jobId=" + id
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
index c8c0e15e0c..916771f573 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
@@ -314,7 +314,8 @@ public class BrokerLoadJob extends BulkLoadJob {
}
RuntimeProfile summaryProfile = new RuntimeProfile("Summary");
- summaryProfile.addInfoString(ProfileManager.QUERY_ID, String.valueOf(id));
+ summaryProfile.addInfoString(ProfileManager.JOB_ID, String.valueOf(this.id));
+ summaryProfile.addInfoString(ProfileManager.QUERY_ID, this.queryId);
summaryProfile.addInfoString(ProfileManager.START_TIME, TimeUtils.longToTimeString(createTimestamp));
summaryProfile.addInfoString(ProfileManager.END_TIME, TimeUtils.longToTimeString(finishTimestamp));
summaryProfile.addInfoString(ProfileManager.TOTAL_TIME,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java
index 07a43fc6e5..a394a62d4a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java
@@ -31,6 +31,7 @@ import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.io.Text;
+import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.common.util.LogBuilder;
import org.apache.doris.common.util.LogKey;
import org.apache.doris.common.util.SqlParserUtils;
@@ -74,6 +75,8 @@ public abstract class BulkLoadJob extends LoadJob {
// input params
protected BrokerDesc brokerDesc;
+ // queryId of OriginStatement
+ protected String queryId;
// this param is used to persist the expr of columns
// the origin stmt is persisted instead of columns expr
// the expr of columns will be reanalyze when the log is replayed
@@ -101,9 +104,11 @@ public abstract class BulkLoadJob extends LoadJob {
this.userInfo = userInfo;
if (ConnectContext.get() != null) {
+ this.queryId = DebugUtil.printId(ConnectContext.get().queryId());
SessionVariable var = ConnectContext.get().getSessionVariable();
sessionVariables.put(SessionVariable.SQL_MODE, Long.toString(var.getSqlMode()));
} else {
+ this.queryId = "N/A";
sessionVariables.put(SessionVariable.SQL_MODE, String.valueOf(SqlModeHelper.MODE_DEFAULT));
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index adeacac51f..a4e2d04f3a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -302,6 +302,7 @@ public class StmtExecutor implements ProfileWriter {
private Map<String, String> getSummaryInfo() {
Map<String, String> infos = Maps.newLinkedHashMap();
+ infos.put(ProfileManager.JOB_ID, "N/A");
infos.put(ProfileManager.QUERY_ID, DebugUtil.printId(context.queryId()));
infos.put(ProfileManager.QUERY_TYPE, queryType);
infos.put(ProfileManager.DORIS_VERSION, Version.DORIS_BUILD_VERSION);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/ExportExportingTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/ExportExportingTask.java
index 8066e280a5..fe6dde27fb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/ExportExportingTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/ExportExportingTask.java
@@ -252,7 +252,8 @@ public class ExportExportingTask extends MasterTask {
private void initProfile() {
profile = new RuntimeProfile("ExportJob");
RuntimeProfile summaryProfile = new RuntimeProfile("Summary");
- summaryProfile.addInfoString(ProfileManager.QUERY_ID, String.valueOf(job.getId()));
+ summaryProfile.addInfoString(ProfileManager.JOB_ID, String.valueOf(job.getId()));
+ summaryProfile.addInfoString(ProfileManager.QUERY_ID, job.getQueryId());
summaryProfile.addInfoString(ProfileManager.START_TIME, TimeUtils.longToTimeString(job.getStartTimeMs()));
long currentTimestamp = System.currentTimeMillis();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org