You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/08/03 01:55:36 UTC
[doris] branch master updated: [improvement](stats) Add more logs and config options (#22436)
This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new fb644ad691 [improvement](stats) Add more logs and config options (#22436)
fb644ad691 is described below
commit fb644ad691702ff654395ec799c4fa04aaf6b8c9
Author: AKIRA <33...@users.noreply.github.com>
AuthorDate: Thu Aug 3 09:55:29 2023 +0800
[improvement](stats) Add more logs and config options (#22436)
1. add more logs and make error messages more clear
2. sleep a while between retry analyze
3. make concurrency of sync analyze configurable
4. Ignore internal columns like delete sign to save resources
---
.../org/apache/doris/analysis/ShowAnalyzeStmt.java | 2 +-
.../doris/nereids/stats/StatsCalculator.java | 24 +++++++------
.../java/org/apache/doris/qe/SessionVariable.java | 5 +++
.../apache/doris/statistics/AnalysisManager.java | 6 +++-
.../apache/doris/statistics/BaseAnalysisTask.java | 2 ++
.../apache/doris/statistics/OlapAnalysisTask.java | 39 ++++++++++------------
.../apache/doris/statistics/StatisticsCache.java | 4 +--
.../doris/statistics/util/StatisticsUtil.java | 8 +++++
.../nereids/jobs/cascades/DeriveStatsJobTest.java | 6 +++-
.../doris/nereids/stats/StatsCalculatorTest.java | 9 +++--
10 files changed, 65 insertions(+), 40 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java
index 3a0aa05a7d..95035641a7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java
@@ -244,7 +244,7 @@ public class ShowAnalyzeStmt extends ShowStmt {
if (!valid) {
throw new AnalysisException("Where clause should looks like: "
- + "STATE = \"PENDING|SCHEDULING|RUNNING|FINISHED|FAILED|CANCELLED\"");
+ + "STATE = \"PENDING|RUNNING|FINISHED|FAILED");
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index 871b102b71..34eef3f433 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -563,13 +563,14 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
}
private Histogram getColumnHistogram(TableIf table, String colName) {
- if (totalHistogramMap.get(table.getName() + colName) != null) {
- return totalHistogramMap.get(table.getName() + colName);
- } else if (isPlayNereidsDump) {
- return null;
- } else {
- return Env.getCurrentEnv().getStatisticsCache().getHistogram(table.getId(), colName);
- }
+ // if (totalHistogramMap.get(table.getName() + colName) != null) {
+ // return totalHistogramMap.get(table.getName() + colName);
+ // } else if (isPlayNereidsDump) {
+ // return null;
+ // } else {
+ // return Env.getCurrentEnv().getStatisticsCache().getHistogram(table.getId(), colName);
+ // }
+ return null;
}
// TODO: 1. Subtract the pruned partition
@@ -583,18 +584,21 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
double rowCount = catalogRelation.getTable().estimatedRowCount();
for (SlotReference slotReference : slotSet) {
String colName = slotReference.getName();
+ boolean shouldIgnoreThisCol = shouldIgnoreCol(table, slotReference.getColumn().get());
+
if (colName == null) {
throw new RuntimeException(String.format("Invalid slot: %s", slotReference.getExprId()));
}
ColumnStatistic cache = Config.enable_stats && FeConstants.enableInternalSchemaDb
- ? getColumnStatistic(table, colName) : ColumnStatistic.UNKNOWN;
+ ? shouldIgnoreThisCol
+ ? ColumnStatistic.UNKNOWN : getColumnStatistic(table, colName) : ColumnStatistic.UNKNOWN;
if (cache.avgSizeByte <= 0) {
cache = new ColumnStatisticBuilder(cache)
.setAvgSizeByte(slotReference.getColumn().get().getType().getSlotSize())
.build();
}
if (cache.isUnKnown) {
- if (forbidUnknownColStats && !ignoreUnknownColStatsCheck(table, slotReference.getColumn().get())) {
+ if (forbidUnknownColStats && !shouldIgnoreThisCol) {
if (StatisticsUtil.statsTblAvailable()) {
throw new AnalysisException(String.format("Found unknown stats for column:%s.%s.\n"
+ "It may caused by:\n"
@@ -1054,7 +1058,7 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
return groupExpression.childStatistics(1);
}
- private boolean ignoreUnknownColStatsCheck(TableIf tableIf, Column c) {
+ private boolean shouldIgnoreCol(TableIf tableIf, Column c) {
if (tableIf instanceof SchemaTable) {
return true;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index a94e5d20a2..9e87681d3d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -359,6 +359,8 @@ public class SessionVariable implements Serializable, Writable {
public static final String ENABLE_STRONG_CONSISTENCY = "enable_strong_consistency_read";
+ public static final String PARALLEL_SYNC_ANALYZE_TASK_NUM = "parallel_sync_analyze_task_num";
+
public static final String CBO_CPU_WEIGHT = "cbo_cpu_weight";
public static final String CBO_MEM_WEIGHT = "cbo_mem_weight";
@@ -1059,6 +1061,9 @@ public class SessionVariable implements Serializable, Writable {
})
public boolean enableStrongConsistencyRead = false;
+ @VariableMgr.VarAttr(name = PARALLEL_SYNC_ANALYZE_TASK_NUM)
+ public int parallelSyncAnalyzeTaskNum = 2;
+
// If this fe is in fuzzy mode, then will use initFuzzyModeVariables to generate some variables,
// not the default value set in the code.
public void initFuzzyModeVariables() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index 63ab923234..9a600a5112 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -751,7 +751,8 @@ public class AnalysisManager extends Daemon implements Writable {
private ThreadPoolExecutor createThreadPoolForSyncAnalyze() {
String poolName = "SYNC ANALYZE THREAD POOL";
- return new ThreadPoolExecutor(0, 64,
+ return new ThreadPoolExecutor(0,
+ ConnectContext.get().getSessionVariable().parallelSyncAnalyzeTaskNum,
0, TimeUnit.SECONDS,
new SynchronousQueue(),
new ThreadFactoryBuilder().setDaemon(true).setNameFormat("SYNC ANALYZE" + "-%d")
@@ -867,6 +868,9 @@ public class AnalysisManager extends Daemon implements Writable {
executor.submit(() -> {
try {
if (cancelled) {
+ errorMessages.add("Cancelled since query timeout,"
+ + "you could set could query_timeout or parallel_sync_analyze_task_num "
+ + "to a bigger value and try again");
return;
}
try {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
index fc264e0661..cab36bc7ea 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
@@ -35,6 +35,7 @@ import org.apache.logging.log4j.Logger;
import java.util.HashSet;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
public abstract class BaseAnalysisTask {
@@ -174,6 +175,7 @@ public abstract class BaseAnalysisTask {
if (retriedTimes > StatisticConstants.ANALYZE_TASK_RETRY_TIMES) {
throw new RuntimeException(t);
}
+ StatisticsUtil.sleep(TimeUnit.SECONDS.toMillis(2 ^ retriedTimes) * 10);
}
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
index 3f3a04d620..04aab26831 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
@@ -50,11 +50,6 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
+ " (SELECT NDV(`${colName}`) AS ndv "
+ " FROM `${dbName}`.`${tblName}` ${sampleExpr}) t2\n";
- @VisibleForTesting
- public OlapAnalysisTask() {
- super();
- }
-
public OlapAnalysisTask(AnalysisInfo info) {
super(info);
}
@@ -110,24 +105,24 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
@VisibleForTesting
public void execSQL(String sql) throws Exception {
- synchronized (OlapAnalysisTask.class) {
- if (killed) {
- return;
- }
- long startTime = System.currentTimeMillis();
- try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext()) {
- r.connectContext.getSessionVariable().disableNereidsPlannerOnce();
- stmtExecutor = new StmtExecutor(r.connectContext, sql);
- r.connectContext.setExecutor(stmtExecutor);
- stmtExecutor.execute();
- QueryState queryState = r.connectContext.getState();
- if (queryState.getStateType().equals(MysqlStateType.ERR)) {
- throw new RuntimeException(String.format("Failed to analyze %s.%s.%s, error: %s sql: %s",
- info.catalogName, info.dbName, info.colName, sql, queryState.getErrorMessage()));
- }
- LOG.info("Analyze SQL: " + sql + " cost time: " + (System.currentTimeMillis() - startTime) + "ms");
+ if (killed) {
+ return;
+ }
+ long startTime = System.currentTimeMillis();
+ LOG.info("ANALYZE SQL : " + sql + " start at " + startTime);
+ try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext()) {
+ r.connectContext.getSessionVariable().disableNereidsPlannerOnce();
+ stmtExecutor = new StmtExecutor(r.connectContext, sql);
+ r.connectContext.setExecutor(stmtExecutor);
+ stmtExecutor.execute();
+ QueryState queryState = r.connectContext.getState();
+ if (queryState.getStateType().equals(MysqlStateType.ERR)) {
+ throw new RuntimeException(String.format("Failed to analyze %s.%s.%s, error: %s sql: %s",
+ info.catalogName, info.dbName, info.colName, sql, queryState.getErrorMessage()));
}
+ } finally {
+ LOG.info("Analyze SQL: " + sql + " cost time: " + (System.currentTimeMillis() - startTime) + "ms");
}
-
}
+
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
index d5a3b972f3..d9328d31a0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
@@ -67,7 +67,6 @@ public class StatisticsCache {
private final AsyncLoadingCache<StatisticsCacheKey, Optional<ColumnStatistic>> columnStatisticsCache =
Caffeine.newBuilder()
.maximumSize(Config.stats_cache_size)
- .expireAfterAccess(Duration.ofHours(StatisticConstants.STATISTICS_CACHE_VALID_DURATION_IN_HOURS))
.refreshAfterWrite(Duration.ofHours(StatisticConstants.STATISTICS_CACHE_REFRESH_INTERVAL))
.executor(threadPool)
.buildAsync(columnStatisticsCacheLoader);
@@ -75,7 +74,6 @@ public class StatisticsCache {
private final AsyncLoadingCache<StatisticsCacheKey, Optional<Histogram>> histogramCache =
Caffeine.newBuilder()
.maximumSize(Config.stats_cache_size)
- .expireAfterAccess(Duration.ofHours(StatisticConstants.STATISTICS_CACHE_VALID_DURATION_IN_HOURS))
.refreshAfterWrite(Duration.ofHours(StatisticConstants.STATISTICS_CACHE_REFRESH_INTERVAL))
.executor(threadPool)
.buildAsync(histogramCacheLoader);
@@ -83,7 +81,7 @@ public class StatisticsCache {
private final AsyncLoadingCache<StatisticsCacheKey, Optional<TableStatistic>> tableStatisticsCache =
Caffeine.newBuilder()
.maximumSize(Config.stats_cache_size)
- .expireAfterWrite(Duration.ofHours(StatisticConstants.ROW_COUNT_CACHE_VALID_DURATION_IN_HOURS))
+ .refreshAfterWrite(Duration.ofHours(StatisticConstants.STATISTICS_CACHE_REFRESH_INTERVAL))
.executor(threadPool)
.buildAsync(tableStatisticsCacheLoader);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index e82a8b955c..0c586a0e62 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -666,4 +666,12 @@ public class StatisticsUtil {
|| type instanceof MapType
|| type instanceof VariantType;
}
+
+ public static void sleep(long millis) {
+ try {
+ Thread.sleep(millis);
+ } catch (InterruptedException ignore) {
+ // IGNORE
+ }
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java
index c6c60c815b..fd60eb2e00 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java
@@ -17,11 +17,14 @@
package org.apache.doris.nereids.jobs.cascades;
+import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
@@ -74,7 +77,8 @@ public class DeriveStatsJobTest {
long tableId1 = 0;
List<String> qualifier = ImmutableList.of("test", "t");
- slot1 = new SlotReference("c1", IntegerType.INSTANCE, true, qualifier);
+ slot1 = new SlotReference(new ExprId(1), "c1", IntegerType.INSTANCE, true, qualifier,
+ new Column("e", PrimitiveType.INT));
new Expectations() {{
ConnectContext.get();
result = context;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
index 4ca19dedf7..0fa8a17863 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
@@ -17,12 +17,15 @@
package org.apache.doris.nereids.stats;
+import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.And;
import org.apache.doris.nereids.trees.expressions.EqualTo;
+import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.SlotReference;
@@ -249,7 +252,8 @@ public class StatsCalculatorTest {
public void testOlapScan(@Mocked ConnectContext context) {
long tableId1 = 0;
List<String> qualifier = ImmutableList.of("test", "t");
- SlotReference slot1 = new SlotReference("c1", IntegerType.INSTANCE, true, qualifier);
+ SlotReference slot1 = new SlotReference(new ExprId(0),
+ "c1", IntegerType.INSTANCE, true, qualifier, new Column("c1", PrimitiveType.INT));
OlapTable table1 = PlanConstructor.newOlapTable(tableId1, "t1", 0);
LogicalOlapScan logicalOlapScan1 = (LogicalOlapScan) new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), table1,
@@ -270,7 +274,8 @@ public class StatsCalculatorTest {
List<String> qualifier = new ArrayList<>();
qualifier.add("test");
qualifier.add("t");
- SlotReference slot1 = new SlotReference("c1", IntegerType.INSTANCE, true, qualifier);
+ SlotReference slot1 = new SlotReference(new ExprId(0),
+ "c1", IntegerType.INSTANCE, true, qualifier, new Column("c1", PrimitiveType.INT));
ColumnStatisticBuilder columnStat1 = new ColumnStatisticBuilder();
columnStat1.setNdv(10);
columnStat1.setNumNulls(5);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org