You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by li...@apache.org on 2024/10/14 14:10:04 UTC
(doris) branch branch-2.1 updated: [Refactor](statistics)Refactor of statistics buildConnectContext. (#41553) (#41814)
This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 19be696120a [Refactor](statistics)Refactor of statistics buildConnectContext. (#41553) (#41814)
19be696120a is described below
commit 19be696120ab1e16df1120717251fadb6a64f030
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Mon Oct 14 22:09:57 2024 +0800
[Refactor](statistics)Refactor of statistics buildConnectContext. (#41553) (#41814)
backport: https://github.com/apache/doris/pull/41553
---
.../doris/datasource/jdbc/JdbcExternalTable.java | 2 +-
.../java/org/apache/doris/qe/AuditLogHelper.java | 18 ++-
.../java/org/apache/doris/qe/ConnectProcessor.java | 2 +-
.../apache/doris/statistics/BaseAnalysisTask.java | 3 +-
.../apache/doris/statistics/OlapAnalysisTask.java | 140 ++++++++++-----------
.../doris/statistics/util/StatisticsUtil.java | 4 +-
.../doris/statistics/OlapAnalysisTaskTest.java | 6 +-
7 files changed, 90 insertions(+), 85 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java
index 07ce183a589..20520d7c542 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java
@@ -119,7 +119,7 @@ public class JdbcExternalTable extends ExternalTable {
params.put("tblName", name);
switch (((JdbcExternalCatalog) catalog).getDatabaseTypeName()) {
case JdbcResource.MYSQL:
- try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext()) {
+ try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext(false)) {
StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
String sql = stringSubstitutor.replace(MYSQL_ROW_COUNT_SQL);
StmtExecutor stmtExecutor = new StmtExecutor(r.connectContext, sql);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java
index ed532a83303..4f221fe42d6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java
@@ -86,18 +86,24 @@ public class AuditLogHelper {
.setFuzzyVariables(!printFuzzyVariables ? "" : ctx.getSessionVariable().printFuzzyVariables());
if (ctx.getState().isQuery()) {
- MetricRepo.COUNTER_QUERY_ALL.increase(1L);
- MetricRepo.USER_COUNTER_QUERY_ALL.getOrAdd(ctx.getQualifiedUser()).increase(1L);
+ if (!ctx.getSessionVariable().internalSession) {
+ MetricRepo.COUNTER_QUERY_ALL.increase(1L);
+ MetricRepo.USER_COUNTER_QUERY_ALL.getOrAdd(ctx.getQualifiedUser()).increase(1L);
+ }
if (ctx.getState().getStateType() == MysqlStateType.ERR
&& ctx.getState().getErrType() != QueryState.ErrType.ANALYSIS_ERR) {
// err query
- MetricRepo.COUNTER_QUERY_ERR.increase(1L);
- MetricRepo.USER_COUNTER_QUERY_ERR.getOrAdd(ctx.getQualifiedUser()).increase(1L);
+ if (!ctx.getSessionVariable().internalSession) {
+ MetricRepo.COUNTER_QUERY_ERR.increase(1L);
+ MetricRepo.USER_COUNTER_QUERY_ERR.getOrAdd(ctx.getQualifiedUser()).increase(1L);
+ }
} else if (ctx.getState().getStateType() == MysqlStateType.OK
|| ctx.getState().getStateType() == MysqlStateType.EOF) {
// ok query
- MetricRepo.HISTO_QUERY_LATENCY.update(elapseMs);
- MetricRepo.USER_HISTO_QUERY_LATENCY.getOrAdd(ctx.getQualifiedUser()).update(elapseMs);
+ if (!ctx.getSessionVariable().internalSession) {
+ MetricRepo.HISTO_QUERY_LATENCY.update(elapseMs);
+ MetricRepo.USER_HISTO_QUERY_LATENCY.getOrAdd(ctx.getQualifiedUser()).update(elapseMs);
+ }
if (elapseMs > Config.qe_slow_log_ms) {
String sqlDigest = DigestUtils.md5Hex(((Queriable) parsedStmt).toDigest());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
index 3a5b35e00d0..d1538284b22 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
@@ -222,7 +222,7 @@ public abstract class ConnectProcessor {
}
public void executeQuery(MysqlCommand mysqlCommand, String originStmt) throws Exception {
- if (MetricRepo.isInit) {
+ if (MetricRepo.isInit && !ctx.getSessionVariable().internalSession) {
MetricRepo.COUNTER_REQUEST_ALL.increase(1L);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
index a7aaaf2c037..31fae23284b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
@@ -290,7 +290,7 @@ public abstract class BaseAnalysisTask {
protected void runQuery(String sql) {
long startTime = System.currentTimeMillis();
String queryId = "";
- try (AutoCloseConnectContext a = StatisticsUtil.buildConnectContext()) {
+ try (AutoCloseConnectContext a = StatisticsUtil.buildConnectContext(info.jobType.equals(JobType.SYSTEM))) {
stmtExecutor = new StmtExecutor(a.connectContext, sql);
ColStatsData colStatsData = new ColStatsData(stmtExecutor.executeInternalQuery().get(0));
// Update index row count after analyze.
@@ -316,5 +316,4 @@ public abstract class BaseAnalysisTask {
stmtExecutor = null;
}
}
-
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
index 5e5d6ac4740..298450b878a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
@@ -30,7 +30,6 @@ import org.apache.doris.common.Pair;
import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.qe.AutoCloseConnectContext;
import org.apache.doris.qe.StmtExecutor;
-import org.apache.doris.statistics.AnalysisInfo.JobType;
import org.apache.doris.statistics.util.StatisticsUtil;
import com.google.common.annotations.VisibleForTesting;
@@ -113,75 +112,73 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
String tabletStr = tabletIds.stream()
.map(Object::toString)
.collect(Collectors.joining(", "));
- try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext(info.jobType.equals(JobType.SYSTEM))) {
- // Get basic stats, including min and max.
- ResultRow basicStats = collectBasicStat(r);
- String min = StatisticsUtil.escapeSQL(basicStats != null && basicStats.getValues().size() > 0
- ? basicStats.get(0) : null);
- String max = StatisticsUtil.escapeSQL(basicStats != null && basicStats.getValues().size() > 1
- ? basicStats.get(1) : null);
+ // Get basic stats, including min and max.
+ ResultRow basicStats = collectBasicStat();
+ String min = StatisticsUtil.escapeSQL(basicStats != null && basicStats.getValues().size() > 0
+ ? basicStats.get(0) : null);
+ String max = StatisticsUtil.escapeSQL(basicStats != null && basicStats.getValues().size() > 1
+ ? basicStats.get(1) : null);
- boolean limitFlag = false;
- long rowsToSample = pair.second;
- Map<String, String> params = new HashMap<>();
- params.put("internalDB", FeConstants.INTERNAL_DB_NAME);
- params.put("columnStatTbl", StatisticConstants.STATISTIC_TBL_NAME);
- params.put("catalogId", String.valueOf(catalog.getId()));
- params.put("catalogName", catalog.getName());
- params.put("dbId", String.valueOf(db.getId()));
- params.put("tblId", String.valueOf(tbl.getId()));
- params.put("idxId", String.valueOf(info.indexId));
- params.put("colId", StatisticsUtil.escapeSQL(String.valueOf(info.colName)));
- params.put("dataSizeFunction", getDataSizeFunction(col, false));
- params.put("dbName", db.getFullName());
- params.put("colName", StatisticsUtil.escapeColumnName(String.valueOf(info.colName)));
- params.put("tblName", String.valueOf(tbl.getName()));
- params.put("scaleFactor", String.valueOf(scaleFactor));
- params.put("sampleHints", tabletStr.isEmpty() ? "" : String.format("TABLET(%s)", tabletStr));
- params.put("ndvFunction", getNdvFunction(String.valueOf(totalRowCount)));
- params.put("min", StatisticsUtil.quote(min));
- params.put("max", StatisticsUtil.quote(max));
- params.put("rowCount", String.valueOf(totalRowCount));
- params.put("type", col.getType().toString());
- params.put("limit", "");
- params.put("index", getIndex());
- if (needLimit()) {
- // If the tablets to be sampled are too large, use limit to control the rows to read, and re-calculate
- // the scaleFactor.
- rowsToSample = Math.min(getSampleRows(), pair.second);
- // Empty table doesn't need to limit.
- if (rowsToSample > 0) {
- limitFlag = true;
- params.put("limit", "limit " + rowsToSample);
- params.put("scaleFactor", String.valueOf(scaleFactor * (double) pair.second / rowsToSample));
- }
+ boolean limitFlag = false;
+ long rowsToSample = pair.second;
+ Map<String, String> params = new HashMap<>();
+ params.put("internalDB", FeConstants.INTERNAL_DB_NAME);
+ params.put("columnStatTbl", StatisticConstants.STATISTIC_TBL_NAME);
+ params.put("catalogId", String.valueOf(catalog.getId()));
+ params.put("catalogName", catalog.getName());
+ params.put("dbId", String.valueOf(db.getId()));
+ params.put("tblId", String.valueOf(tbl.getId()));
+ params.put("idxId", String.valueOf(info.indexId));
+ params.put("colId", StatisticsUtil.escapeSQL(String.valueOf(info.colName)));
+ params.put("dataSizeFunction", getDataSizeFunction(col, false));
+ params.put("dbName", db.getFullName());
+ params.put("colName", StatisticsUtil.escapeColumnName(String.valueOf(info.colName)));
+ params.put("tblName", String.valueOf(tbl.getName()));
+ params.put("scaleFactor", String.valueOf(scaleFactor));
+ params.put("sampleHints", tabletStr.isEmpty() ? "" : String.format("TABLET(%s)", tabletStr));
+ params.put("ndvFunction", getNdvFunction(String.valueOf(totalRowCount)));
+ params.put("min", StatisticsUtil.quote(min));
+ params.put("max", StatisticsUtil.quote(max));
+ params.put("rowCount", String.valueOf(totalRowCount));
+ params.put("type", col.getType().toString());
+ params.put("limit", "");
+ params.put("index", getIndex());
+ if (needLimit()) {
+ // If the tablets to be sampled are too large, use limit to control the rows to read, and re-calculate
+ // the scaleFactor.
+ rowsToSample = Math.min(getSampleRows(), pair.second);
+ // Empty table doesn't need to limit.
+ if (rowsToSample > 0) {
+ limitFlag = true;
+ params.put("limit", "limit " + rowsToSample);
+ params.put("scaleFactor", String.valueOf(scaleFactor * (double) pair.second / rowsToSample));
}
- StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
- String sql;
- if (useLinearAnalyzeTemplate()) {
- // For single unique key, use count as ndv.
- if (isSingleUniqueKey()) {
- params.put("ndvFunction", String.valueOf(totalRowCount));
- } else {
- params.put("ndvFunction", "ROUND(NDV(`${colName}`) * ${scaleFactor})");
- }
- sql = stringSubstitutor.replace(LINEAR_ANALYZE_TEMPLATE);
+ }
+ StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
+ String sql;
+ if (useLinearAnalyzeTemplate()) {
+ // For single unique key, use count as ndv.
+ if (isSingleUniqueKey()) {
+ params.put("ndvFunction", String.valueOf(totalRowCount));
} else {
- params.put("dataSizeFunction", getDataSizeFunction(col, true));
- params.put("subStringColName", getStringTypeColName(col));
- sql = stringSubstitutor.replace(DUJ1_ANALYZE_TEMPLATE);
+ params.put("ndvFunction", "ROUND(NDV(`${colName}`) * ${scaleFactor})");
}
- LOG.info("Sample for column [{}]. Total rows [{}], rows to sample [{}], scale factor [{}], "
- + "limited [{}], distribute column [{}], partition column [{}], key column [{}], "
- + "is single unique key [{}]",
- col.getName(), params.get("rowCount"), rowsToSample, params.get("scaleFactor"),
- limitFlag, tbl.isDistributionColumn(col.getName()),
- tbl.isPartitionColumn(col.getName()), col.isKey(), isSingleUniqueKey());
- runQuery(sql);
+ sql = stringSubstitutor.replace(LINEAR_ANALYZE_TEMPLATE);
+ } else {
+ params.put("dataSizeFunction", getDataSizeFunction(col, true));
+ params.put("subStringColName", getStringTypeColName(col));
+ sql = stringSubstitutor.replace(DUJ1_ANALYZE_TEMPLATE);
}
+ LOG.info("Sample for column [{}]. Total rows [{}], rows to sample [{}], scale factor [{}], "
+ + "limited [{}], distribute column [{}], partition column [{}], key column [{}], "
+ + "is single unique key [{}]",
+ col.getName(), params.get("rowCount"), rowsToSample, params.get("scaleFactor"),
+ limitFlag, tbl.isDistributionColumn(col.getName()),
+ tbl.isPartitionColumn(col.getName()), col.isKey(), isSingleUniqueKey());
+ runQuery(sql);
}
- protected ResultRow collectBasicStat(AutoCloseConnectContext context) {
+ protected ResultRow collectBasicStat() {
// Agg table value columns has no zone map.
// For these columns, skip collecting min and max value to avoid scan whole table.
if (((OlapTable) tbl).getKeysType().equals(KeysType.AGG_KEYS) && !col.isKey()) {
@@ -197,14 +194,17 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
params.put("index", getIndex());
StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
String sql = stringSubstitutor.replace(BASIC_STATS_TEMPLATE);
- stmtExecutor = new StmtExecutor(context.connectContext, sql);
- ResultRow resultRow = stmtExecutor.executeInternalQuery().get(0);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Cost time in millisec: " + (System.currentTimeMillis() - startTime)
- + " Min max SQL: " + sql + " QueryId: " + DebugUtil.printId(stmtExecutor.getContext().queryId()));
+ ResultRow resultRow = null;
+ try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext(false)) {
+ stmtExecutor = new StmtExecutor(r.connectContext, sql);
+ resultRow = stmtExecutor.executeInternalQuery().get(0);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Cost time in millisec: " + (System.currentTimeMillis() - startTime) + " Min max SQL: "
+ + sql + " QueryId: " + DebugUtil.printId(stmtExecutor.getContext().queryId()));
+ }
+ // Release the reference to stmtExecutor, reduce memory usage.
+ stmtExecutor = null;
}
- // Release the reference to stmtExecutor, reduce memory usage.
- stmtExecutor = null;
return resultRow;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index dc1dba5d981..25563bf6c39 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -142,7 +142,7 @@ public class StatisticsUtil {
public static QueryState execUpdate(String sql) throws Exception {
StmtExecutor stmtExecutor = null;
- AutoCloseConnectContext r = StatisticsUtil.buildConnectContext();
+ AutoCloseConnectContext r = StatisticsUtil.buildConnectContext(false);
try {
r.connectContext.getSessionVariable().disableNereidsPlannerOnce();
stmtExecutor = new StmtExecutor(r.connectContext, sql);
@@ -209,7 +209,7 @@ public class StatisticsUtil {
}
public static void analyze(StatementBase statementBase) throws UserException {
- try (AutoCloseConnectContext r = buildConnectContext()) {
+ try (AutoCloseConnectContext r = buildConnectContext(false)) {
Analyzer analyzer = new Analyzer(Env.getCurrentEnv(), r.connectContext);
statementBase.analyze(analyzer);
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
index 5bb0920e433..a7283e351de 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
@@ -141,7 +141,7 @@ public class OlapAnalysisTaskTest {
}
@Mock
- public ResultRow collectBasicStat(AutoCloseConnectContext context) {
+ public ResultRow collectBasicStat() {
List<String> values = Lists.newArrayList();
values.add("1");
values.add("2");
@@ -221,7 +221,7 @@ public class OlapAnalysisTaskTest {
}
@Mock
- public ResultRow collectBasicStat(AutoCloseConnectContext context) {
+ public ResultRow collectBasicStat() {
List<String> values = Lists.newArrayList();
values.add("1");
values.add("2");
@@ -303,7 +303,7 @@ public class OlapAnalysisTaskTest {
}
@Mock
- public ResultRow collectBasicStat(AutoCloseConnectContext context) {
+ public ResultRow collectBasicStat() {
List<String> values = Lists.newArrayList();
values.add("1");
values.add("2");
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org