You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/10/13 04:56:38 UTC
[doris] branch branch-2.0 updated: Revert "[refactor][backport](stats) Migrate stats framework from master to branch 2.0 (#25119)" (#25403)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 470d9ab03cc Revert "[refactor][backport](stats) Migrate stats framework from master to branch 2.0 (#25119)" (#25403)
470d9ab03cc is described below
commit 470d9ab03cce593d9a6a5096fa541abf55cff6db
Author: AKIRA <33...@users.noreply.github.com>
AuthorDate: Fri Oct 13 12:56:28 2023 +0800
Revert "[refactor][backport](stats) Migrate stats framework from master to branch 2.0 (#25119)" (#25403)
This reverts commit 4b6ff32ef079a1332aae7f62695e530ec3bc977d.
Co-authored-by: AKIHA <cy...@example.com>
---
.../main/java/org/apache/doris/common/Config.java | 7 -
fe/fe-core/src/main/cup/sql_parser.cup | 12 +-
.../apache/doris/analysis/AnalyzeProperties.java | 54 +-
.../org/apache/doris/analysis/AnalyzeStmt.java | 13 +-
.../org/apache/doris/analysis/AnalyzeTblStmt.java | 54 +-
.../org/apache/doris/analysis/PartitionNames.java | 38 +-
.../org/apache/doris/analysis/ShowAnalyzeStmt.java | 84 ++-
.../apache/doris/analysis/ShowTableStatsStmt.java | 39 +-
.../main/java/org/apache/doris/catalog/Env.java | 14 +-
.../doris/catalog/InternalSchemaInitializer.java | 40 +-
.../java/org/apache/doris/catalog/OlapTable.java | 53 --
.../main/java/org/apache/doris/catalog/Table.java | 11 -
.../java/org/apache/doris/catalog/TableIf.java | 13 -
.../doris/catalog/external/ExternalTable.java | 21 -
.../doris/catalog/external/HMSExternalTable.java | 59 +-
.../doris/catalog/external/JdbcExternalTable.java | 18 +-
.../org/apache/doris/common/ThreadPoolManager.java | 9 -
.../org/apache/doris/datasource/CatalogIf.java | 3 -
.../org/apache/doris/datasource/CatalogMgr.java | 6 -
.../apache/doris/datasource/ExternalCatalog.java | 16 -
.../apache/doris/datasource/InternalCatalog.java | 5 -
.../org/apache/doris/journal/JournalEntity.java | 17 -
.../org/apache/doris/nereids/cost/CostModelV1.java | 63 +-
.../java/org/apache/doris/nereids/cost/CostV1.java | 30 +-
.../processor/post/RuntimeFilterPruner.java | 21 +-
.../doris/nereids/stats/ExpressionEstimation.java | 54 +-
.../doris/nereids/stats/FilterEstimation.java | 379 ++++------
.../apache/doris/nereids/stats/JoinEstimation.java | 63 +-
.../doris/nereids/stats/StatsCalculator.java | 44 +-
.../nereids/trees/expressions/Properties.java | 96 ---
.../trees/expressions/functions/table/Hdfs.java | 8 +-
.../trees/expressions/functions/table/Local.java | 6 +-
.../trees/expressions/functions/table/Numbers.java | 10 +-
.../trees/expressions/functions/table/S3.java | 8 +-
.../functions/table/TableValuedFunction.java | 8 +-
.../expressions/visitor/ExpressionVisitor.java | 5 -
.../org/apache/doris/nereids/types/MapType.java | 2 -
.../doris/nereids/types/coercion/AnyDataType.java | 18 +-
.../java/org/apache/doris/persist/EditLog.java | 19 +-
.../doris/persist/TableStatsDeletionLog.java | 47 --
.../java/org/apache/doris/qe/AuditLogHelper.java | 120 ----
.../main/java/org/apache/doris/qe/DdlExecutor.java | 3 +
.../doris/qe/InternalQueryExecutionException.java | 24 -
.../java/org/apache/doris/qe/SessionVariable.java | 20 -
.../java/org/apache/doris/qe/ShowExecutor.java | 37 +-
.../java/org/apache/doris/qe/StmtExecutor.java | 35 +-
.../org/apache/doris/statistics/AnalysisInfo.java | 116 +--
.../doris/statistics/AnalysisInfoBuilder.java | 39 +-
.../apache/doris/statistics/AnalysisManager.java | 789 +++++++++------------
.../org/apache/doris/statistics/AnalysisState.java | 2 -
.../doris/statistics/AnalysisTaskExecutor.java | 53 +-
.../doris/statistics/AnalysisTaskScheduler.java | 108 +++
.../doris/statistics/AnalysisTaskWrapper.java | 12 -
.../apache/doris/statistics/BaseAnalysisTask.java | 61 +-
.../org/apache/doris/statistics/ColStatsData.java | 82 ---
.../org/apache/doris/statistics/ColStatsMeta.java | 58 --
.../apache/doris/statistics/ColumnStatistic.java | 144 ++--
.../doris/statistics/ColumnStatisticBuilder.java | 47 +-
.../statistics/ColumnStatisticsCacheLoader.java | 62 +-
.../apache/doris/statistics/HMSAnalysisTask.java | 254 +++----
.../java/org/apache/doris/statistics/HistData.java | 36 -
.../org/apache/doris/statistics/Histogram.java | 18 +-
.../doris/statistics/HistogramCacheLoader.java | 1 +
.../apache/doris/statistics/JdbcAnalysisTask.java | 9 +-
.../apache/doris/statistics/OlapAnalysisJob.java | 56 --
.../apache/doris/statistics/OlapAnalysisTask.java | 116 +--
.../org/apache/doris/statistics/ResultRow.java | 59 --
.../doris/statistics/StatisticConstants.java | 54 +-
.../apache/doris/statistics/StatisticRange.java | 96 +--
.../apache/doris/statistics/StatisticalType.java | 2 -
.../org/apache/doris/statistics/Statistics.java | 133 +++-
.../doris/statistics/StatisticsAutoAnalyzer.java | 215 ++++++
.../doris/statistics/StatisticsAutoCollector.java | 196 -----
.../apache/doris/statistics/StatisticsBuilder.java | 2 +-
.../apache/doris/statistics/StatisticsCache.java | 158 ++---
.../apache/doris/statistics/StatisticsCleaner.java | 21 +-
.../doris/statistics/StatisticsCollector.java | 86 ---
.../statistics/StatisticsPeriodCollector.java | 50 --
.../doris/statistics/StatisticsRepository.java | 144 +++-
.../apache/doris/statistics/StatsDeriveResult.java | 7 +
.../java/org/apache/doris/statistics/StatsId.java | 58 --
.../apache/doris/statistics/TableStatistic.java | 61 ++
.../doris/statistics/TableStatisticBuilder.java | 51 ++
.../statistics/TableStatisticsCacheLoader.java | 60 ++
.../apache/doris/statistics/TableStatsMeta.java | 137 ----
.../apache/doris/statistics/TaskStatusWrapper.java | 33 -
.../doris/statistics/util/InternalQuery.java | 21 +-
.../doris/statistics/util/InternalQueryResult.java | 242 +++++++
.../apache/doris/statistics/util/SimpleQueue.java | 65 --
.../doris/statistics/util/StatisticsUtil.java | 163 +----
.../jobs/joinorder/hypergraph/OtherJoinTest.java | 31 +-
.../doris/nereids/stats/FilterEstimationTest.java | 85 ++-
.../doris/nereids/util/HyperGraphBuilder.java | 51 +-
.../apache/doris/statistics/AnalysisJobTest.java | 33 +-
.../doris/statistics/AnalysisTaskExecutorTest.java | 37 +-
.../org/apache/doris/statistics/CacheTest.java | 192 ++---
.../apache/doris/statistics/HistogramTaskTest.java | 14 +-
.../doris/statistics/StatsDeriveResultTest.java | 5 +-
.../org/apache/doris/statistics/StatsMockUtil.java | 48 --
.../statistics/util/InternalQueryResultTest.java | 119 ++++
gensrc/thrift/FrontendService.thrift | 2 +-
.../suites/statistics/analyze_stats.groovy | 84 +--
102 files changed, 2724 insertions(+), 3860 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 72ecebca781..dae88cf40f4 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2178,13 +2178,6 @@ public class Config extends ConfigBase {
})
public static int autobucket_min_buckets = 1;
- @ConfField
- public static int full_auto_analyze_simultaneously_running_task_num = 1;
-
- @ConfField
- public static final int period_analyze_simultaneously_running_task_num = 1;
-
-
@ConfField(mutable = true, description = {
"Doris 为了兼用 mysql 周边工具生态，会内置一个名为 mysql 的数据库，如果该数据库与用户自建数据库冲突，"
+ "请修改这个字段，为 doris 内置的 mysql database 更换一个名字",
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup
index 7f3aa2020e8..5950548c427 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -4134,17 +4134,13 @@ show_param ::=
RESULT = new ShowCreateMaterializedViewStmt(mvName, tableName);
:}
/* show analyze job */
- | KW_ANALYZE opt_table_name:tbl opt_wild_where
+ | KW_ANALYZE opt_table_name:tbl opt_wild_where order_by_clause:orderByClause limit_clause:limitClause
{:
- RESULT = new ShowAnalyzeStmt(tbl, parser.where, false);
+ RESULT = new ShowAnalyzeStmt(tbl, parser.where, orderByClause, limitClause);
:}
- | KW_ANALYZE INTEGER_LITERAL:jobId opt_wild_where
+ | KW_ANALYZE INTEGER_LITERAL:jobId opt_wild_where order_by_clause:orderByClause limit_clause:limitClause
{:
- RESULT = new ShowAnalyzeStmt(jobId, parser.where);
- :}
- | KW_AUTO KW_ANALYZE opt_table_name:tbl opt_wild_where
- {:
- RESULT = new ShowAnalyzeStmt(tbl, parser.where, true);
+ RESULT = new ShowAnalyzeStmt(jobId, parser.where, orderByClause, limitClause);
:}
| KW_ANALYZE KW_TASK KW_STATUS INTEGER_LITERAL:jobId
{:
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeProperties.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeProperties.java
index d7e639da3a5..ccb122bc269 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeProperties.java
@@ -22,18 +22,16 @@ import org.apache.doris.common.util.PrintableMap;
import org.apache.doris.statistics.AnalysisInfo.AnalysisType;
import com.google.common.collect.ImmutableSet;
-import com.google.gson.annotations.SerializedName;
import org.apache.commons.lang3.StringUtils;
-import org.apache.logging.log4j.core.util.CronExpression;
-import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
-// TODO: Remove map
public class AnalyzeProperties {
+ private final Map<String, String> properties;
+
public static final String PROPERTY_SYNC = "sync";
public static final String PROPERTY_INCREMENTAL = "incremental";
public static final String PROPERTY_AUTOMATIC = "automatic";
@@ -43,23 +41,6 @@ public class AnalyzeProperties {
public static final String PROPERTY_ANALYSIS_TYPE = "analysis.type";
public static final String PROPERTY_PERIOD_SECONDS = "period.seconds";
- public static final String PROPERTY_FORCE_FULL = "force.full";
-
- public static final AnalyzeProperties DEFAULT_PROP = new AnalyzeProperties(new HashMap<String, String>() {
- {
- put(AnalyzeProperties.PROPERTY_SYNC, "false");
- put(AnalyzeProperties.PROPERTY_AUTOMATIC, "false");
- put(AnalyzeProperties.PROPERTY_ANALYSIS_TYPE, AnalysisType.FUNDAMENTALS.toString());
- }
- });
-
- public static final String PROPERTY_PERIOD_CRON = "period.cron";
-
- private CronExpression cronExpression;
-
- @SerializedName("analyzeProperties")
- private final Map<String, String> properties;
-
private static final ImmutableSet<String> PROPERTIES_SET = new ImmutableSet.Builder<String>()
.add(PROPERTY_SYNC)
.add(PROPERTY_INCREMENTAL)
@@ -69,8 +50,6 @@ public class AnalyzeProperties {
.add(PROPERTY_NUM_BUCKETS)
.add(PROPERTY_ANALYSIS_TYPE)
.add(PROPERTY_PERIOD_SECONDS)
- .add(PROPERTY_PERIOD_CRON)
- .add(PROPERTY_FORCE_FULL)
.build();
public AnalyzeProperties(Map<String, String> properties) {
@@ -93,7 +72,6 @@ public class AnalyzeProperties {
checkAnalysisMode(msgTemplate);
checkAnalysisType(msgTemplate);
checkScheduleType(msgTemplate);
- checkPeriod();
}
public boolean isSync() {
@@ -137,10 +115,6 @@ public class AnalyzeProperties {
return TimeUnit.SECONDS.toMillis(minutes);
}
- public CronExpression getCron() {
- return cronExpression;
- }
-
private void checkPeriodSeconds() throws AnalysisException {
if (properties.containsKey(PROPERTY_PERIOD_SECONDS)) {
checkNumericProperty(PROPERTY_PERIOD_SECONDS, properties.get(PROPERTY_PERIOD_SECONDS),
@@ -233,22 +207,6 @@ public class AnalyzeProperties {
}
}
- private void checkPeriod() throws AnalysisException {
- if (properties.containsKey(PROPERTY_PERIOD_SECONDS)
- && properties.containsKey(PROPERTY_PERIOD_CRON)) {
- throw new AnalysisException(PROPERTY_PERIOD_SECONDS + " and " + PROPERTY_PERIOD_CRON
- + " couldn't be set simultaneously");
- }
- String cronExprStr = properties.get(PROPERTY_PERIOD_CRON);
- if (cronExprStr != null) {
- try {
- cronExpression = new CronExpression(cronExprStr);
- } catch (java.text.ParseException e) {
- throw new AnalysisException("Invalid cron expression: " + cronExprStr);
- }
- }
- }
-
private void checkNumericProperty(String key, String value, int lowerBound, int upperBound,
boolean includeBoundary, String errorMsg) throws AnalysisException {
if (!StringUtils.isNumeric(value)) {
@@ -268,14 +226,6 @@ public class AnalyzeProperties {
|| properties.containsKey(PROPERTY_SAMPLE_ROWS);
}
- public boolean forceFull() {
- return properties.containsKey(PROPERTY_FORCE_FULL);
- }
-
- public boolean isSampleRows() {
- return properties.containsKey(PROPERTY_SAMPLE_ROWS);
- }
-
public String toSQL() {
StringBuilder sb = new StringBuilder();
sb.append("PROPERTIES(");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeStmt.java
index ae2c6a7ff48..6f1f7c64d8f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeStmt.java
@@ -23,8 +23,6 @@ import org.apache.doris.statistics.AnalysisInfo.AnalysisMode;
import org.apache.doris.statistics.AnalysisInfo.AnalysisType;
import org.apache.doris.statistics.AnalysisInfo.ScheduleType;
-import org.apache.logging.log4j.core.util.CronExpression;
-
import java.util.Map;
public class AnalyzeStmt extends StatementBase {
@@ -57,8 +55,7 @@ public class AnalyzeStmt extends StatementBase {
if (analyzeProperties.isAutomatic()) {
return ScheduleType.AUTOMATIC;
}
- return analyzeProperties.getPeriodTimeInMs() > 0 || analyzeProperties.getCron() != null
- ? ScheduleType.PERIOD : ScheduleType.ONCE;
+ return analyzeProperties.getPeriodTimeInMs() > 0 ? ScheduleType.PERIOD : ScheduleType.ONCE;
}
public boolean isSync() {
@@ -89,12 +86,4 @@ public class AnalyzeStmt extends StatementBase {
public RedirectStatus getRedirectStatus() {
return RedirectStatus.FORWARD_WITH_SYNC;
}
-
- public CronExpression getCron() {
- return analyzeProperties.getCron();
- }
-
- public boolean forceFull() {
- return analyzeProperties.forceFull();
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java
index cbc66f367f2..920b60627a8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java
@@ -24,7 +24,6 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.View;
-import org.apache.doris.catalog.external.ExternalTable;
import org.apache.doris.catalog.external.HMSExternalTable;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
@@ -42,7 +41,6 @@ import org.apache.doris.statistics.util.StatisticsUtil;
import com.google.common.collect.Sets;
import org.apache.commons.lang3.StringUtils;
-import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.Set;
@@ -86,7 +84,7 @@ public class AnalyzeTblStmt extends AnalyzeStmt {
private final TableName tableName;
private List<String> columnNames;
- private PartitionNames partitionNames;
+ private List<String> partitionNames;
private boolean isAllColumns;
// after analyzed
@@ -99,7 +97,7 @@ public class AnalyzeTblStmt extends AnalyzeStmt {
AnalyzeProperties properties) {
super(properties);
this.tableName = tableName;
- this.partitionNames = partitionNames;
+ this.partitionNames = partitionNames == null ? null : partitionNames.getPartitionNames();
this.columnNames = columnNames;
this.analyzeProperties = properties;
this.isAllColumns = columnNames == null;
@@ -168,9 +166,11 @@ public class AnalyzeTblStmt extends AnalyzeStmt {
analyzeProperties.check();
// TODO support external table
- if (analyzeProperties.isSampleRows() && !(table instanceof OlapTable)) {
- throw new AnalysisException("Sampling statistics "
- + "collection of external tables is not supported with rows, use percent instead.");
+ if (analyzeProperties.isSample()) {
+ if (!(table instanceof OlapTable)) {
+ throw new AnalysisException("Sampling statistics "
+ + "collection of external tables is not supported");
+ }
}
if (analyzeProperties.isSync()
&& (analyzeProperties.isAutomatic() || analyzeProperties.getPeriodTimeInMs() != 0)) {
@@ -181,9 +181,6 @@ public class AnalyzeTblStmt extends AnalyzeStmt {
throw new AnalysisException("Automatic collection "
+ "and period statistics collection cannot be set at same time");
}
- if (analyzeProperties.isSample() && analyzeProperties.forceFull()) {
- throw new AnalysisException("Impossible to analyze with sample and full simultaneously");
- }
}
private void checkColumn() throws AnalysisException {
@@ -199,8 +196,7 @@ public class AnalyzeTblStmt extends AnalyzeStmt {
}
}
if (containsUnsupportedTytpe) {
- if (ConnectContext.get() == null
- || !ConnectContext.get().getSessionVariable().enableAnalyzeComplexTypeColumn) {
+ if (!ConnectContext.get().getSessionVariable().enableAnalyzeComplexTypeColumn) {
columnNames = columnNames.stream()
.filter(c -> !StatisticsUtil.isUnsupportedType(table.getColumn(c).getType()))
.collect(Collectors.toList());
@@ -240,33 +236,14 @@ public class AnalyzeTblStmt extends AnalyzeStmt {
}
public Set<String> getPartitionNames() {
- if (partitionNames == null || partitionNames.getPartitionNames() == null) {
- if (table instanceof ExternalTable) {
- // External table couldn't return all partitions when partitionNames is not set.
- // Because Analyze Table command for external table could specify partition names.
- return Collections.emptySet();
- }
- return table.getPartitionNames();
+ Set<String> partitions = partitionNames == null ? table.getPartitionNames() : Sets.newHashSet(partitionNames);
+ if (isSamplingPartition()) {
+ int partNum = ConnectContext.get().getSessionVariable().getExternalTableAnalyzePartNum();
+ partitions = partitions.stream().limit(partNum).collect(Collectors.toSet());
}
- Set<String> partitions = Sets.newHashSet();
- partitions.addAll(partitionNames.getPartitionNames());
return partitions;
}
- public boolean isAllPartitions() {
- if (partitionNames == null) {
- return false;
- }
- return partitionNames.isAllPartitions();
- }
-
- public long getPartitionCount() {
- if (partitionNames == null) {
- return 0;
- }
- return partitionNames.getCount();
- }
-
public boolean isPartitionOnly() {
return partitionNames != null;
}
@@ -283,13 +260,8 @@ public class AnalyzeTblStmt extends AnalyzeStmt {
}
private void checkAnalyzePriv(String dbName, String tblName) throws AnalysisException {
- ConnectContext ctx = ConnectContext.get();
- // means it a system analyze
- if (ctx == null) {
- return;
- }
if (!Env.getCurrentEnv().getAccessManager()
- .checkTblPriv(ctx, dbName, tblName, PrivPredicate.SELECT)) {
+ .checkTblPriv(ConnectContext.get(), dbName, tblName, PrivPredicate.SELECT)) {
ErrorReport.reportAnalysisException(
ErrorCode.ERR_TABLEACCESS_DENIED_ERROR,
"ANALYZE",
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionNames.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionNames.java
index ca26a2978e0..1140dfc6777 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionNames.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionNames.java
@@ -48,37 +48,15 @@ public class PartitionNames implements ParseNode, Writable {
// true if these partitions are temp partitions
@SerializedName(value = "isTemp")
private final boolean isTemp;
- private final boolean allPartitions;
- private final long count;
- // Default partition count to collect statistic for external table.
- private static final long DEFAULT_PARTITION_COUNT = 100;
public PartitionNames(boolean isTemp, List<String> partitionNames) {
this.partitionNames = partitionNames;
this.isTemp = isTemp;
- this.allPartitions = false;
- this.count = 0;
}
public PartitionNames(PartitionNames other) {
this.partitionNames = Lists.newArrayList(other.partitionNames);
this.isTemp = other.isTemp;
- this.allPartitions = other.allPartitions;
- this.count = 0;
- }
-
- public PartitionNames(boolean allPartitions) {
- this.partitionNames = null;
- this.isTemp = false;
- this.allPartitions = allPartitions;
- this.count = 0;
- }
-
- public PartitionNames(long partitionCount) {
- this.partitionNames = null;
- this.isTemp = false;
- this.allPartitions = false;
- this.count = partitionCount;
}
public List<String> getPartitionNames() {
@@ -89,23 +67,9 @@ public class PartitionNames implements ParseNode, Writable {
return isTemp;
}
- public boolean isAllPartitions() {
- return allPartitions;
- }
-
- public long getCount() {
- return count;
- }
-
@Override
public void analyze(Analyzer analyzer) throws AnalysisException {
- if (allPartitions && count > 0) {
- throw new AnalysisException("All partition and partition count couldn't be set at the same time.");
- }
- if (allPartitions || count > 0) {
- return;
- }
- if (partitionNames == null || partitionNames.isEmpty()) {
+ if (partitionNames.isEmpty()) {
throw new AnalysisException("No partition specified in partition lists");
}
// check if partition name is not empty string
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java
index fb19cb2fd5b..95035641a7e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java
@@ -25,6 +25,7 @@ import org.apache.doris.common.Config;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
+import org.apache.doris.common.util.OrderByPair;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowResultSetMetaData;
@@ -34,6 +35,10 @@ import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.IntStream;
+
/**
* ShowAnalyzeStmt is used to show statistics job info.
* syntax:
@@ -64,30 +69,36 @@ public class ShowAnalyzeStmt extends ShowStmt {
.build();
private long jobId;
- private final TableName dbTableName;
- private final Expr whereClause;
-
- // extract from predicate
+ private TableName dbTableName;
+ private Expr whereClause;
+ private LimitElement limitElement;
+ private List<OrderByElement> orderByElements;
private String stateValue;
+ private ArrayList<OrderByPair> orderByPairs;
- private final boolean auto;
-
+ public ShowAnalyzeStmt() {
+ }
public ShowAnalyzeStmt(TableName dbTableName,
- Expr whereClause, boolean auto) {
+ Expr whereClause,
+ List<OrderByElement> orderByElements,
+ LimitElement limitElement) {
this.dbTableName = dbTableName;
this.whereClause = whereClause;
- this.auto = auto;
-
+ this.orderByElements = orderByElements;
+ this.limitElement = limitElement;
}
public ShowAnalyzeStmt(long jobId,
- Expr whereClause) {
+ Expr whereClause,
+ List<OrderByElement> orderByElements,
+ LimitElement limitElement) {
Preconditions.checkArgument(jobId > 0, "JobId must greater than 0.");
this.jobId = jobId;
this.dbTableName = null;
this.whereClause = whereClause;
- this.auto = false;
+ this.orderByElements = orderByElements;
+ this.limitElement = limitElement;
}
public long getJobId() {
@@ -100,6 +111,12 @@ public class ShowAnalyzeStmt extends ShowStmt {
return stateValue;
}
+ public ArrayList<OrderByPair> getOrderByPairs() {
+ Preconditions.checkArgument(isAnalyzed(),
+ "The orderByPairs must be obtained after the parsing is complete");
+ return orderByPairs;
+ }
+
public Expr getWhereClause() {
Preconditions.checkArgument(isAnalyzed(),
"The whereClause must be obtained after the parsing is complete");
@@ -107,6 +124,13 @@ public class ShowAnalyzeStmt extends ShowStmt {
return whereClause;
}
+ public long getLimit() {
+ if (limitElement != null && limitElement.hasLimit()) {
+ return limitElement.getLimit();
+ }
+ return -1L;
+ }
+
@Override
public void analyze(Analyzer analyzer) throws UserException {
if (!Config.enable_stats) {
@@ -125,6 +149,21 @@ public class ShowAnalyzeStmt extends ShowStmt {
if (whereClause != null) {
analyzeSubPredicate(whereClause);
}
+
+ // analyze order by
+ if (orderByElements != null && !orderByElements.isEmpty()) {
+ orderByPairs = new ArrayList<>();
+ for (OrderByElement orderByElement : orderByElements) {
+ if (orderByElement.getExpr() instanceof SlotRef) {
+ SlotRef slotRef = (SlotRef) orderByElement.getExpr();
+ int index = analyzeColumn(slotRef.getColumnName());
+ OrderByPair orderByPair = new OrderByPair(index, !orderByElement.getIsAsc());
+ orderByPairs.add(orderByPair);
+ } else {
+ throw new AnalysisException("Should order by column");
+ }
+ }
+ }
}
@Override
@@ -240,6 +279,25 @@ public class ShowAnalyzeStmt extends ShowStmt {
sb.append(whereClause.toSql());
}
+ // Order By clause
+ if (orderByElements != null) {
+ sb.append(" ");
+ sb.append("ORDER BY");
+ sb.append(" ");
+ IntStream.range(0, orderByElements.size()).forEach(i -> {
+ sb.append(orderByElements.get(i).getExpr().toSql());
+ sb.append((orderByElements.get(i).getIsAsc()) ? " ASC" : " DESC");
+ sb.append((i + 1 != orderByElements.size()) ? ", " : "");
+ });
+ }
+
+ if (getLimit() != -1L) {
+ sb.append(" ");
+ sb.append("LIMIT");
+ sb.append(" ");
+ sb.append(getLimit());
+ }
+
return sb.toString();
}
@@ -251,8 +309,4 @@ public class ShowAnalyzeStmt extends ShowStmt {
public TableName getDbTableName() {
return dbTableName;
}
-
- public boolean isAuto() {
- return auto;
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java
index fe499fa1b08..da10d5c492b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java
@@ -32,13 +32,12 @@ import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowResultSet;
import org.apache.doris.qe.ShowResultSetMetaData;
-import org.apache.doris.statistics.TableStatsMeta;
+import org.apache.doris.statistics.TableStatistic;
+import org.apache.doris.statistics.util.StatisticsUtil;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
-import java.sql.Date;
-import java.util.ArrayList;
import java.util.List;
public class ShowTableStatsStmt extends ShowStmt {
@@ -46,12 +45,9 @@ public class ShowTableStatsStmt extends ShowStmt {
// TODO add more columns
private static final ImmutableList<String> TITLE_NAMES =
new ImmutableList.Builder<String>()
- .add("updated_rows")
- .add("query_times")
.add("row_count")
- .add("updated_time")
- .add("columns")
- .add("trigger")
+ .add("update_time")
+ .add("last_analyze_time")
.build();
private final TableName tableName;
@@ -130,33 +126,12 @@ public class ShowTableStatsStmt extends ShowStmt {
return table.getPartition(partitionName).getId();
}
- public ShowResultSet constructResultSet(TableStatsMeta tableStatistic) {
- if (tableStatistic == null) {
- return new ShowResultSet(getMetaData(), new ArrayList<>());
- }
+ public ShowResultSet constructResultSet(TableStatistic tableStatistic) {
List<List<String>> result = Lists.newArrayList();
List<String> row = Lists.newArrayList();
- row.add(String.valueOf(tableStatistic.updatedRows));
- row.add(String.valueOf(tableStatistic.queriedTimes.get()));
row.add(String.valueOf(tableStatistic.rowCount));
- row.add(new Date(tableStatistic.updatedTime).toString());
- row.add(tableStatistic.analyzeColumns().toString());
- row.add(tableStatistic.jobType.toString());
- result.add(row);
- return new ShowResultSet(getMetaData(), result);
- }
-
- public ShowResultSet constructResultSet(long rowCount) {
- List<List<String>> result = Lists.newArrayList();
- List<String> row = Lists.newArrayList();
- row.add("");
- row.add("");
- row.add(String.valueOf(rowCount));
- row.add("");
- row.add("");
- row.add("");
- row.add("");
- row.add("");
+ row.add(String.valueOf(tableStatistic.updateTime));
+ row.add(StatisticsUtil.getReadableTime(tableStatistic.lastAnalyzeTimeInMs));
result.add(row);
return new ShowResultSet(getMetaData(), result);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 89b63455590..ea4410f60c6 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -211,7 +211,7 @@ import org.apache.doris.resource.Tag;
import org.apache.doris.resource.workloadgroup.WorkloadGroupMgr;
import org.apache.doris.service.FrontendOptions;
import org.apache.doris.statistics.AnalysisManager;
-import org.apache.doris.statistics.StatisticsAutoCollector;
+import org.apache.doris.statistics.StatisticsAutoAnalyzer;
import org.apache.doris.statistics.StatisticsCache;
import org.apache.doris.statistics.StatisticsCleaner;
import org.apache.doris.statistics.query.QueryStats;
@@ -457,7 +457,7 @@ public class Env {
*/
private final LoadManagerAdapter loadManagerAdapter;
- private StatisticsAutoCollector statisticsAutoCollector;
+ private StatisticsAutoAnalyzer statisticsAutoAnalyzer;
private HiveTransactionMgr hiveTransactionMgr;
@@ -663,7 +663,7 @@ public class Env {
this.extMetaCacheMgr = new ExternalMetaCacheMgr();
this.analysisManager = new AnalysisManager();
this.statisticsCleaner = new StatisticsCleaner();
- this.statisticsAutoCollector = new StatisticsAutoCollector();
+ this.statisticsAutoAnalyzer = new StatisticsAutoAnalyzer();
this.globalFunctionMgr = new GlobalFunctionMgr();
this.workloadGroupMgr = new WorkloadGroupMgr();
this.queryStats = new QueryStats();
@@ -907,8 +907,8 @@ public class Env {
if (statisticsCleaner != null) {
statisticsCleaner.start();
}
- if (statisticsAutoCollector != null) {
- statisticsAutoCollector.start();
+ if (statisticsAutoAnalyzer != null) {
+ statisticsAutoAnalyzer.start();
}
}
@@ -5421,8 +5421,8 @@ public class Env {
return loadManagerAdapter;
}
- public StatisticsAutoCollector getStatisticsAutoCollector() {
- return statisticsAutoCollector;
+ public StatisticsAutoAnalyzer getStatisticsAutoAnalyzer() {
+ return statisticsAutoAnalyzer;
}
public QueryStats getQueryStats() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java
index b7fdec73f02..fd42de00f78 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java
@@ -83,6 +83,7 @@ public class InternalSchemaInitializer extends Thread {
return;
}
Database database = op.get();
+ modifyTblReplicaCount(database, StatisticConstants.ANALYSIS_TBL_NAME);
modifyTblReplicaCount(database, StatisticConstants.STATISTIC_TBL_NAME);
modifyTblReplicaCount(database, StatisticConstants.HISTOGRAM_TBL_NAME);
}
@@ -125,6 +126,7 @@ public class InternalSchemaInitializer extends Thread {
}
private void createTbl() throws UserException {
+ Env.getCurrentEnv().getInternalCatalog().createTable(buildAnalysisTblStmt());
Env.getCurrentEnv().getInternalCatalog().createTable(buildStatisticsTblStmt());
Env.getCurrentEnv().getInternalCatalog().createTable(buildHistogramTblStmt());
}
@@ -143,6 +145,41 @@ public class InternalSchemaInitializer extends Thread {
}
}
+ @VisibleForTesting
+ public CreateTableStmt buildAnalysisTblStmt() throws UserException {
+ TableName tableName = new TableName("",
+ FeConstants.INTERNAL_DB_NAME, StatisticConstants.ANALYSIS_TBL_NAME);
+ List<ColumnDef> columnDefs = new ArrayList<>();
+ columnDefs.add(new ColumnDef("id", TypeDef.createVarchar(StatisticConstants.ID_LEN)));
+ columnDefs.add(new ColumnDef("catalog_id", TypeDef.createVarchar(StatisticConstants.MAX_NAME_LEN)));
+ columnDefs.add(new ColumnDef("db_id", TypeDef.createVarchar(StatisticConstants.MAX_NAME_LEN)));
+ columnDefs.add(new ColumnDef("tbl_id", TypeDef.createVarchar(StatisticConstants.MAX_NAME_LEN)));
+ columnDefs.add(new ColumnDef("idx_id", TypeDef.createVarchar(StatisticConstants.MAX_NAME_LEN)));
+ ColumnDef partId = new ColumnDef("part_id", TypeDef.createVarchar(StatisticConstants.MAX_NAME_LEN));
+ partId.setAllowNull(true);
+ columnDefs.add(partId);
+ columnDefs.add(new ColumnDef("count", TypeDef.create(PrimitiveType.BIGINT)));
+ columnDefs.add(new ColumnDef("last_analyze_time_in_ms", TypeDef.create(PrimitiveType.BIGINT)));
+ columnDefs.add(new ColumnDef("update_time", TypeDef.create(PrimitiveType.DATETIME)));
+ String engineName = "olap";
+ ArrayList<String> uniqueKeys = Lists.newArrayList("id", "catalog_id",
+ "db_id", "tbl_id", "idx_id", "part_id");
+ KeysDesc keysDesc = new KeysDesc(KeysType.UNIQUE_KEYS, uniqueKeys);
+ DistributionDesc distributionDesc = new HashDistributionDesc(
+ StatisticConstants.STATISTIC_TABLE_BUCKET_COUNT, uniqueKeys);
+ Map<String, String> properties = new HashMap<String, String>() {
+ {
+ put("replication_num", String.valueOf(
+ Math.max(1, Config.min_replication_num_per_tablet)));
+ }
+ };
+ CreateTableStmt createTableStmt = new CreateTableStmt(true, false,
+ tableName, columnDefs, engineName, keysDesc, null, distributionDesc,
+ properties, null, "Doris internal statistics table, DO NOT MODIFY IT", null);
+ StatisticsUtil.analyze(createTableStmt);
+ return createTableStmt;
+ }
+
@VisibleForTesting
public CreateTableStmt buildStatisticsTblStmt() throws UserException {
TableName tableName = new TableName("",
@@ -244,7 +281,8 @@ public class InternalSchemaInitializer extends Thread {
}
return false;
}
- return db.getTable(StatisticConstants.HISTOGRAM_TBL_NAME).isPresent();
+ return db.getTable(StatisticConstants.HISTOGRAM_TBL_NAME).isPresent()
+ && db.getTable(StatisticConstants.ANALYSIS_TBL_NAME).isPresent();
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index f59df2554d3..9975ba0230a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -55,8 +55,6 @@ import org.apache.doris.statistics.BaseAnalysisTask;
import org.apache.doris.statistics.HistogramTask;
import org.apache.doris.statistics.MVAnalysisTask;
import org.apache.doris.statistics.OlapAnalysisTask;
-import org.apache.doris.statistics.TableStatsMeta;
-import org.apache.doris.statistics.util.StatisticsUtil;
import org.apache.doris.system.Backend;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.TColumn;
@@ -2251,55 +2249,4 @@ public class OlapTable extends Table {
}
}
}
-
- @Override
- public Map<String, Set<String>> findReAnalyzeNeededPartitions() {
- TableIf table = this;
- TableStatsMeta tableStats = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(table.getId());
- Set<String> allPartitions = table.getPartitionNames().stream().map(table::getPartition)
- .filter(Partition::hasData).map(Partition::getName).collect(Collectors.toSet());
- if (tableStats == null) {
- return table.getBaseSchema().stream().collect(Collectors.toMap(Column::getName, v -> allPartitions));
- }
- Map<String, Set<String>> colToPart = new HashMap<>();
- for (Column col : table.getBaseSchema()) {
- long lastUpdateTime = tableStats.findColumnLastUpdateTime(col.getName());
- Set<String> partitions = table.getPartitionNames().stream()
- .map(table::getPartition)
- .filter(Partition::hasData)
- .filter(partition ->
- partition.getVisibleVersionTime() >= lastUpdateTime).map(Partition::getName)
- .collect(Collectors.toSet());
- colToPart.put(col.getName(), partitions);
- }
- return colToPart;
- }
-
- public long getDataSize(boolean singleReplica) {
- long dataSize = 0;
- for (Partition partition : getAllPartitions()) {
- dataSize += partition.getDataSize(singleReplica);
- }
- return dataSize;
- }
-
- public boolean needReAnalyzeTable(TableStatsMeta tblStats) {
- if (tblStats == null) {
- return true;
- }
- long rowCount = getRowCount();
- // TODO: Do we need to analyze an empty table?
- if (rowCount == 0) {
- return false;
- }
- if (!tblStats.analyzeColumns().containsAll(getBaseSchema()
- .stream()
- .map(Column::getName)
- .collect(Collectors.toSet()))) {
- return true;
- }
- long updateRows = tblStats.updatedRows.get();
- int tblHealth = StatisticsUtil.getTableHealth(rowCount, updateRows);
- return tblHealth < Config.table_stats_health_threshold;
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
index ba7e55c7d86..0c50fc42b4b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
@@ -30,7 +30,6 @@ import org.apache.doris.external.hudi.HudiTable;
import org.apache.doris.statistics.AnalysisInfo;
import org.apache.doris.statistics.BaseAnalysisTask;
import org.apache.doris.statistics.ColumnStatistic;
-import org.apache.doris.statistics.TableStatsMeta;
import org.apache.doris.thrift.TTableDescriptor;
import com.google.common.base.Preconditions;
@@ -558,14 +557,4 @@ public abstract class Table extends MetaObject implements Writable, TableIf {
public void analyze(String dbName) {
}
-
- @Override
- public Map<String, Set<String>> findReAnalyzeNeededPartitions() {
- return Collections.emptyMap();
- }
-
- @Override
- public boolean needReAnalyzeTable(TableStatsMeta tblStats) {
- return true;
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
index 108d227e591..78717f0eca7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
@@ -23,7 +23,6 @@ import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.statistics.AnalysisInfo;
import org.apache.doris.statistics.BaseAnalysisTask;
import org.apache.doris.statistics.ColumnStatistic;
-import org.apache.doris.statistics.TableStatsMeta;
import org.apache.doris.thrift.TTableDescriptor;
import com.google.common.collect.Lists;
@@ -34,7 +33,6 @@ import java.io.DataOutput;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
-import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -238,16 +236,5 @@ public interface TableIf {
default long getLastUpdateTime() {
return -1L;
}
-
- Map<String, Set<String>> findReAnalyzeNeededPartitions();
-
- default long getDataSize(boolean singleReplica) {
- // TODO: Each tableIf should impl it by itself.
- return 0;
- }
-
- boolean needReAnalyzeTable(TableStatsMeta tblStats);
-
-
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java
index 6a1630e297c..f4c76cda7a9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java
@@ -35,10 +35,8 @@ import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.statistics.AnalysisInfo;
import org.apache.doris.statistics.BaseAnalysisTask;
import org.apache.doris.statistics.ColumnStatistic;
-import org.apache.doris.statistics.TableStatsMeta;
import org.apache.doris.thrift.TTableDescriptor;
-import com.google.common.collect.Sets;
import com.google.gson.annotations.SerializedName;
import lombok.Getter;
import org.apache.commons.lang3.NotImplementedException;
@@ -48,14 +46,10 @@ import org.apache.logging.log4j.Logger;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.util.HashSet;
import java.util.List;
-import java.util.Map;
import java.util.Optional;
-import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
/**
* External table represent tables that are not self-managed by Doris.
@@ -379,19 +373,4 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable {
rwLock = new ReentrantReadWriteLock(true);
objectCreated = false;
}
-
- @Override
- public boolean needReAnalyzeTable(TableStatsMeta tblStats) {
- // TODO: Find a way to decide if this external table need to be reanalyzed.
- // For now, simply return true for all external tables.
- return true;
- }
-
- @Override
- public Map<String, Set<String>> findReAnalyzeNeededPartitions() {
- HashSet<String> partitions = Sets.newHashSet();
- // TODO: Find a way to collect external table partitions that need to be analyzed.
- partitions.add("Dummy Partition");
- return getBaseSchema().stream().collect(Collectors.toMap(Column::getName, k -> partitions));
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
index 5852bf69aae..02242803666 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
@@ -32,7 +32,7 @@ import org.apache.doris.statistics.BaseAnalysisTask;
import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.ColumnStatisticBuilder;
import org.apache.doris.statistics.HMSAnalysisTask;
-import org.apache.doris.statistics.TableStatsMeta;
+import org.apache.doris.statistics.TableStatistic;
import org.apache.doris.statistics.util.StatisticsUtil;
import org.apache.doris.thrift.THiveTable;
import org.apache.doris.thrift.TTableDescriptor;
@@ -57,7 +57,6 @@ import org.apache.iceberg.Table;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.time.LocalDate;
@@ -103,16 +102,13 @@ public class HMSExternalTable extends ExternalTable {
SUPPORTED_HUDI_FILE_FORMATS.add("com.uber.hoodie.hadoop.realtime.HoodieRealtimeInputFormat");
}
- protected volatile org.apache.hadoop.hive.metastore.api.Table remoteTable = null;
- protected List<Column> partitionColumns;
+ private volatile org.apache.hadoop.hive.metastore.api.Table remoteTable = null;
+ private List<Column> partitionColumns;
- protected DLAType dlaType = DLAType.UNKNOWN;
-
- // No as precise as row count in TableStats, but better than none.
- private long estimatedRowCount = -1;
+ private DLAType dlaType = DLAType.UNKNOWN;
public enum DLAType {
- UNKNOWN, HIVE, HUDI, ICEBERG, DELTALAKE
+ UNKNOWN, HIVE, HUDI, ICEBERG
}
/**
@@ -127,10 +123,6 @@ public class HMSExternalTable extends ExternalTable {
super(id, name, catalog, dbName, TableType.HMS_EXTERNAL_TABLE);
}
- public HMSExternalTable(long id, String name, String dbName, HMSExternalCatalog catalog, TableType type) {
- super(id, name, catalog, dbName, type);
- }
-
public boolean isSupportedHmsTable() {
makeSureInitialized();
return dlaType != DLAType.UNKNOWN;
@@ -154,7 +146,6 @@ public class HMSExternalTable extends ExternalTable {
}
}
objectCreated = true;
- estimatedRowCount = getRowCountFromExternalSource(true);
}
}
@@ -278,19 +269,10 @@ public class HMSExternalTable extends ExternalTable {
@Override
public long getRowCount() {
makeSureInitialized();
- long rowCount = getRowCountFromExternalSource(false);
- if (rowCount == -1) {
- LOG.debug("Will estimate row count from file list.");
- rowCount = StatisticsUtil.getRowCountFromFileList(this);
- }
- return rowCount;
- }
-
- private long getRowCountFromExternalSource(boolean isInit) {
long rowCount;
switch (dlaType) {
case HIVE:
- rowCount = StatisticsUtil.getHiveRowCount(this, isInit);
+ rowCount = StatisticsUtil.getHiveRowCount(this);
break;
case ICEBERG:
rowCount = StatisticsUtil.getIcebergRowCount(this);
@@ -299,6 +281,10 @@ public class HMSExternalTable extends ExternalTable {
LOG.warn("getRowCount for dlaType {} is not supported.", dlaType);
rowCount = -1;
}
+ if (rowCount == -1) {
+ LOG.debug("Will estimate row count from file list.");
+ rowCount = StatisticsUtil.getRowCountFromFileList(this);
+ }
return rowCount;
}
@@ -436,20 +422,13 @@ public class HMSExternalTable extends ExternalTable {
@Override
public long estimatedRowCount() {
try {
- TableStatsMeta tableStats = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(id);
- if (tableStats != null) {
- long rowCount = tableStats.rowCount;
+ Optional<TableStatistic> tableStatistics = Env.getCurrentEnv().getStatisticsCache().getTableStatistics(
+ catalog.getId(), catalog.getDbOrAnalysisException(dbName).getId(), id);
+ if (tableStatistics.isPresent()) {
+ long rowCount = tableStatistics.get().rowCount;
LOG.debug("Estimated row count for db {} table {} is {}.", dbName, name, rowCount);
return rowCount;
}
-
- if (estimatedRowCount != -1) {
- return estimatedRowCount;
- }
- // Cache the estimated row count in this structure
- // though the table never get analyzed, since the row estimation might be expensive caused by RPC.
- estimatedRowCount = getRowCount();
- return estimatedRowCount;
} catch (Exception e) {
LOG.warn("Fail to get row count for table {}", name, e);
}
@@ -470,7 +449,7 @@ public class HMSExternalTable extends ExternalTable {
return tmpSchema;
}
- protected void initPartitionColumns(List<Column> schema) {
+ private void initPartitionColumns(List<Column> schema) {
List<String> partitionKeys = remoteTable.getPartitionKeys().stream().map(FieldSchema::getName)
.collect(Collectors.toList());
partitionColumns = Lists.newArrayListWithCapacity(partitionKeys.size());
@@ -501,7 +480,7 @@ public class HMSExternalTable extends ExternalTable {
return getHiveColumnStats(colName);
case ICEBERG:
return StatisticsUtil.getIcebergColumnStats(colName,
- Env.getCurrentEnv().getExtMetaCacheMgr().getIcebergMetadataCache().getIcebergTable(this));
+ Env.getCurrentEnv().getExtMetaCacheMgr().getIcebergMetadataCache().getIcebergTable(this));
default:
LOG.warn("get column stats for dlaType {} is not supported.", dlaType);
}
@@ -629,12 +608,6 @@ public class HMSExternalTable extends ExternalTable {
builder.setMaxValue(Double.MAX_VALUE);
}
}
-
- @Override
- public void gsonPostProcess() throws IOException {
- super.gsonPostProcess();
- estimatedRowCount = -1;
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java
index a02c59080fc..051bfa5e585 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java
@@ -24,13 +24,14 @@ import org.apache.doris.datasource.jdbc.JdbcExternalCatalog;
import org.apache.doris.statistics.AnalysisInfo;
import org.apache.doris.statistics.BaseAnalysisTask;
import org.apache.doris.statistics.JdbcAnalysisTask;
-import org.apache.doris.statistics.TableStatsMeta;
+import org.apache.doris.statistics.TableStatistic;
import org.apache.doris.thrift.TTableDescriptor;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.List;
+import java.util.Optional;
/**
* Elasticsearch external table.
@@ -111,11 +112,16 @@ public class JdbcExternalTable extends ExternalTable {
@Override
public long getRowCount() {
makeSureInitialized();
- TableStatsMeta tableStats = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(id);
- if (tableStats != null) {
- long rowCount = tableStats.rowCount;
- LOG.debug("Estimated row count for db {} table {} is {}.", dbName, name, rowCount);
- return rowCount;
+ try {
+ Optional<TableStatistic> tableStatistics = Env.getCurrentEnv().getStatisticsCache().getTableStatistics(
+ catalog.getId(), catalog.getDbOrAnalysisException(dbName).getId(), id);
+ if (tableStatistics.isPresent()) {
+ long rowCount = tableStatistics.get().rowCount;
+ LOG.debug("Estimated row count for db {} table {} is {}.", dbName, name, rowCount);
+ return rowCount;
+ }
+ } catch (Exception e) {
+ LOG.warn("Fail to get row count for table {}", name, e);
}
return 1;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ThreadPoolManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/ThreadPoolManager.java
index 31d608b8a25..be8731b6b25 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/ThreadPoolManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/ThreadPoolManager.java
@@ -134,15 +134,6 @@ public class ThreadPoolManager {
poolName, needRegisterMetric);
}
- public static ThreadPoolExecutor newDaemonFixedThreadPool(int numThread, int queueSize,
- String poolName,
- boolean needRegisterMetric,
- RejectedExecutionHandler handler) {
- return newDaemonThreadPool(numThread, numThread, KEEP_ALIVE_TIME, TimeUnit.SECONDS,
- new LinkedBlockingQueue<>(queueSize), handler,
- poolName, needRegisterMetric);
- }
-
public static <T> ThreadPoolExecutor newDaemonFixedPriorityThreadPool(int numThread, int initQueueSize,
Comparator<T> comparator, Class<T> tClass,
String poolName, boolean needRegisterMetric) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
index cebc526d14b..2d8925b5d98 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
@@ -175,7 +175,4 @@ public interface CatalogIf<T extends DatabaseIf> {
public Collection<DatabaseIf> getAllDbs();
public ConcurrentHashMap<Long, DatabaseIf> getIdToDb();
-
- public boolean enableAutoAnalyze();
-
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
index e13372254a1..02a14076855 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -63,10 +63,8 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.Arrays;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Function;
@@ -1149,9 +1147,5 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
public Map<Long, CatalogIf> getIdToCatalog() {
return idToCatalog;
}
-
- public Set<CatalogIf> getCopyOfCatalog() {
- return new HashSet<>(idToCatalog.values());
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 22a6816543e..499a1f9385f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -74,8 +74,6 @@ public abstract class ExternalCatalog
implements CatalogIf<ExternalDatabase<? extends ExternalTable>>, Writable, GsonPostProcessable {
private static final Logger LOG = LogManager.getLogger(ExternalCatalog.class);
- public static final String ENABLE_AUTO_ANALYZE = "enable.auto.analyze";
-
// Unique id of this catalog, will be assigned after catalog is loaded.
@SerializedName(value = "id")
protected long id;
@@ -599,18 +597,4 @@ public abstract class ExternalCatalog
public ConcurrentHashMap<Long, DatabaseIf> getIdToDb() {
return new ConcurrentHashMap<>(idToDb);
}
-
- @Override
- public boolean enableAutoAnalyze() {
- // By default, external catalog disables auto analyze, uses could set catalog property to enable it:
- // "enable.auto.analyze" = true
- Map<String, String> properties = catalogProperty.getProperties();
- boolean ret = false;
- if (properties.containsKey(ENABLE_AUTO_ANALYZE)
- && properties.get(ENABLE_AUTO_ANALYZE).equalsIgnoreCase("true")) {
- ret = true;
- }
- return ret;
- }
-
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index aee9b5f67af..65c8f885604 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -3183,9 +3183,4 @@ public class InternalCatalog implements CatalogIf<Database> {
public Collection<DatabaseIf> getAllDbs() {
return new HashSet<>(idToDb.values());
}
-
- @Override
- public boolean enableAutoAnalyze() {
- return true;
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
index 62b4bcfc851..0e0b8ac7df6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
@@ -111,7 +111,6 @@ import org.apache.doris.persist.TableAddOrDropInvertedIndicesInfo;
import org.apache.doris.persist.TableInfo;
import org.apache.doris.persist.TablePropertyInfo;
import org.apache.doris.persist.TableRenameColumnInfo;
-import org.apache.doris.persist.TableStatsDeletionLog;
import org.apache.doris.persist.TruncateTableInfo;
import org.apache.doris.plugin.PluginInfo;
import org.apache.doris.policy.DropPolicyLog;
@@ -119,7 +118,6 @@ import org.apache.doris.policy.Policy;
import org.apache.doris.policy.StoragePolicy;
import org.apache.doris.resource.workloadgroup.WorkloadGroup;
import org.apache.doris.statistics.AnalysisInfo;
-import org.apache.doris.statistics.TableStatsMeta;
import org.apache.doris.system.Backend;
import org.apache.doris.system.Frontend;
import org.apache.doris.transaction.TransactionState;
@@ -847,21 +845,6 @@ public class JournalEntity implements Writable {
isRead = true;
break;
}
- case OperationType.OP_UPDATE_TABLE_STATS: {
- data = TableStatsMeta.read(in);
- isRead = true;
- break;
- }
- case OperationType.OP_PERSIST_AUTO_JOB: {
- data = AnalysisInfo.read(in);
- isRead = true;
- break;
- }
- case OperationType.OP_DELETE_TABLE_STATS: {
- data = TableStatsDeletionLog.read(in);
- isRead = true;
- break;
- }
default: {
IOException e = new IOException();
LOG.error("UNKNOWN Operation Type {}", opCode, e);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
index aa8f4d6cc7c..1f7255b7990 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
@@ -86,7 +86,8 @@ class CostModelV1 extends PlanVisitor<Cost, PlanContext> {
CostV1 planCostV1 = (CostV1) planCost;
return new CostV1(childCostV1.getCpuCost() + planCostV1.getCpuCost(),
childCostV1.getMemoryCost() + planCostV1.getMemoryCost(),
- childCostV1.getNetworkCost() + planCostV1.getNetworkCost());
+ childCostV1.getNetworkCost() + planCostV1.getNetworkCost(),
+ childCostV1.getPenalty() + planCostV1.getPenalty());
}
@Override
@@ -117,7 +118,7 @@ class CostModelV1 extends PlanVisitor<Cost, PlanContext> {
CostV1 costValue = (CostV1) storageLayerAggregate.getRelation().accept(this, context);
// multiply a factor less than 1, so we can select PhysicalStorageLayerAggregate as far as possible
return new CostV1(costValue.getCpuCost() * 0.7, costValue.getMemoryCost(),
- costValue.getNetworkCost());
+ costValue.getNetworkCost(), costValue.getPenalty());
}
@Override
@@ -149,14 +150,14 @@ class CostModelV1 extends PlanVisitor<Cost, PlanContext> {
// TODO: consider two-phase sort and enforcer.
Statistics statistics = context.getStatisticsWithCheck();
Statistics childStatistics = context.getChildStatistics(0);
-
- double childRowCount = childStatistics.getRowCount();
- double rowCount = statistics.getRowCount();
if (physicalQuickSort.getSortPhase().isGather()) {
// Now we do more like two-phase sort, so penalise one-phase sort
- rowCount *= 100;
+ statistics = statistics.withRowCount(statistics.getRowCount() * 100);
}
- return CostV1.of(childRowCount, rowCount, childRowCount);
+ return CostV1.of(
+ childStatistics.getRowCount(),
+ statistics.getRowCount(),
+ childStatistics.getRowCount());
}
@Override
@@ -164,14 +165,14 @@ class CostModelV1 extends PlanVisitor<Cost, PlanContext> {
// TODO: consider two-phase sort and enforcer.
Statistics statistics = context.getStatisticsWithCheck();
Statistics childStatistics = context.getChildStatistics(0);
-
- double childRowCount = childStatistics.getRowCount();
- double rowCount = statistics.getRowCount();
if (topN.getSortPhase().isGather()) {
// Now we do more like two-phase sort, so penalise one-phase sort
- rowCount *= 100;
+ statistics = statistics.withRowCount(statistics.getRowCount() * 100);
}
- return CostV1.of(childRowCount, rowCount, childRowCount);
+ return CostV1.of(
+ childStatistics.getRowCount(),
+ statistics.getRowCount(),
+ childStatistics.getRowCount());
}
@Override
@@ -185,9 +186,9 @@ class CostModelV1 extends PlanVisitor<Cost, PlanContext> {
Statistics statistics = context.getStatisticsWithCheck();
Statistics childStatistics = context.getChildStatistics(0);
return CostV1.of(
- childStatistics.getRowCount(),
- statistics.getRowCount(),
- childStatistics.getRowCount());
+ childStatistics.getRowCount(),
+ statistics.getRowCount(),
+ childStatistics.getRowCount());
}
@Override
@@ -286,38 +287,30 @@ class CostModelV1 extends PlanVisitor<Cost, PlanContext> {
pattern2: (L join1 Agg1) join2 agg2
in pattern2, join1 and join2 takes more time, but Agg1 and agg2 can be processed in parallel.
*/
+ double penalty = HEAVY_OPERATOR_PUNISH_FACTOR
+ * Math.min(probeStats.getPenalty(), buildStats.getPenalty());
+ if (buildStats.getWidth() >= 2) {
+ //penalty for right deep tree
+ penalty += rightRowCount;
+ }
if (physicalHashJoin.getJoinType().isCrossJoin()) {
return CostV1.of(leftRowCount + rightRowCount + outputRowCount,
0,
- leftRowCount + rightRowCount
- );
+ leftRowCount + rightRowCount,
+ penalty);
}
if (context.isBroadcastJoin()) {
- // compared with shuffle join, bc join will be taken a penalty for both build and probe side;
- // currently we use the following factor as the penalty factor:
- // build side factor: totalInstanceNumber to the power of 2, standing for the additional effort for
- // bigger cost for building hash table, taken on rightRowCount
- // probe side factor: totalInstanceNumber to the power of 2, standing for the additional effort for
- // bigger cost for ProbeWhenBuildSideOutput effort and ProbeWhenSearchHashTableTime
- // on the output rows, taken on outputRowCount()
- double probeSideFactor = 1.0;
- double buildSideFactor = ConnectContext.get().getSessionVariable().getBroadcastRightTableScaleFactor();
- int parallelInstance = Math.max(1, ConnectContext.get().getSessionVariable().getParallelExecInstanceNum());
- int totalInstanceNumber = parallelInstance * beNumber;
- if (buildSideFactor <= 1.0) {
- // use totalInstanceNumber to the power of 2 as the default factor value
- buildSideFactor = Math.pow(totalInstanceNumber, 0.5);
- }
- // TODO: since the outputs rows may expand a lot, penalty on it will cause bc never be chosen.
- // will refine this in next generation cost model.
- return CostV1.of(leftRowCount + rightRowCount * buildSideFactor + outputRowCount * probeSideFactor,
+ double broadcastJoinPenalty = broadCastJoinBalancePenalty(probeStats, buildStats);
+ return CostV1.of(leftRowCount * broadcastJoinPenalty + rightRowCount + outputRowCount,
rightRowCount,
+ 0,
0
);
}
return CostV1.of(leftRowCount + rightRowCount + outputRowCount,
rightRowCount,
+ 0,
0
);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostV1.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostV1.java
index bf1cc425999..b5c5b50bd2e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostV1.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostV1.java
@@ -19,19 +19,23 @@ package org.apache.doris.nereids.cost;
class CostV1 implements Cost {
private static final CostV1 INFINITE = new CostV1(Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY,
- Double.POSITIVE_INFINITY);
- private static final CostV1 ZERO = new CostV1(0, 0, 0);
+ Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY);
+ private static final CostV1 ZERO = new CostV1(0, 0, 0, 0);
private final double cpuCost;
private final double memoryCost;
private final double networkCost;
+ //penalty for
+ // 1. right deep tree
+ // 2. right XXX join
+ private final double penalty;
private final double cost;
/**
* Constructor of CostEstimate.
*/
- public CostV1(double cpuCost, double memoryCost, double networkCost) {
+ public CostV1(double cpuCost, double memoryCost, double networkCost, double penaltiy) {
// TODO: fix stats
cpuCost = Double.max(0, cpuCost);
memoryCost = Double.max(0, memoryCost);
@@ -39,10 +43,11 @@ class CostV1 implements Cost {
this.cpuCost = cpuCost;
this.memoryCost = memoryCost;
this.networkCost = networkCost;
+ this.penalty = penaltiy;
CostWeight costWeight = CostWeight.get();
this.cost = costWeight.cpuWeight * cpuCost + costWeight.memoryWeight * memoryCost
- + costWeight.networkWeight * networkCost;
+ + costWeight.networkWeight * networkCost + costWeight.penaltyWeight * penalty;
}
public CostV1(double cost) {
@@ -50,6 +55,7 @@ class CostV1 implements Cost {
this.cpuCost = 0;
this.networkCost = 0;
this.memoryCost = 0;
+ this.penalty = 0;
}
public static CostV1 infinite() {
@@ -72,20 +78,28 @@ class CostV1 implements Cost {
return networkCost;
}
+ public double getPenalty() {
+ return penalty;
+ }
+
public double getValue() {
return cost;
}
+ public static CostV1 of(double cpuCost, double maxMemory, double networkCost, double rightDeepPenaltiy) {
+ return new CostV1(cpuCost, maxMemory, networkCost, rightDeepPenaltiy);
+ }
+
public static CostV1 of(double cpuCost, double maxMemory, double networkCost) {
- return new CostV1(cpuCost, maxMemory, networkCost);
+ return new CostV1(cpuCost, maxMemory, networkCost, 0);
}
public static CostV1 ofCpu(double cpuCost) {
- return new CostV1(cpuCost, 0, 0);
+ return new CostV1(cpuCost, 0, 0, 0);
}
public static CostV1 ofMemory(double memoryCost) {
- return new CostV1(0, memoryCost, 0);
+ return new CostV1(0, memoryCost, 0, 0);
}
@Override
@@ -93,7 +107,7 @@ class CostV1 implements Cost {
StringBuilder sb = new StringBuilder();
sb.append("[").append((long) cpuCost).append("/")
.append((long) memoryCost).append("/").append((long) networkCost)
- .append("/").append("]");
+ .append("/").append((long) penalty).append("]");
return sb.toString();
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java
index c9cc43d0c29..aa1f10aa478 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java
@@ -27,6 +27,7 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows;
import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute;
import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate;
import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
@@ -53,6 +54,17 @@ import java.util.Set;
*/
public class RuntimeFilterPruner extends PlanPostProcessor {
+ // *******************************
+ // Physical plans
+ // *******************************
+ @Override
+ public PhysicalHashAggregate visitPhysicalHashAggregate(
+ PhysicalHashAggregate<? extends Plan> agg, CascadesContext context) {
+ agg.child().accept(this, context);
+ context.getRuntimeFilterContext().addEffectiveSrcNode(agg);
+ return agg;
+ }
+
@Override
public PhysicalQuickSort visitPhysicalQuickSort(PhysicalQuickSort<? extends Plan> sort, CascadesContext context) {
sort.child().accept(this, context);
@@ -153,9 +165,7 @@ public class RuntimeFilterPruner extends PlanPostProcessor {
/**
* consider L join R on L.a=R.b
* runtime-filter: L.a<-R.b is effective,
- * if rf could reduce tuples of L,
- * 1. some L.a distinctive value are not covered by R.b, or
- * 2. if there is a effective RF applied on R
+ * if R.b.selectivity<1 or b is partly covered by a
*
* TODO: min-max
* @param equalTo join condition
@@ -189,7 +199,8 @@ public class RuntimeFilterPruner extends PlanPostProcessor {
if (probeColumnStat.isUnKnown || buildColumnStat.isUnKnown) {
return true;
}
- double buildNdvInProbeRange = buildColumnStat.ndvIntersection(probeColumnStat);
- return probeColumnStat.ndv > buildNdvInProbeRange * (1 + ColumnStatistic.STATS_ERROR);
+ return buildColumnStat.selectivity < 1
+ || probeColumnStat.notEnclosed(buildColumnStat)
+ || buildColumnStat.ndv < probeColumnStat.ndv * 0.95;
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java
index aa1903e7b37..b8a7975a087 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java
@@ -151,7 +151,7 @@ public class ExpressionEstimation extends ExpressionVisitor<ColumnStatistic, Sta
return new ColumnStatisticBuilder()
.setNdv(2)
.setMinValue(0)
- .setMaxValue(Double.POSITIVE_INFINITY)
+ .setMaxValue(Double.MAX_VALUE)
.setAvgSizeByte(8)
.setNumNulls(0)
.build();
@@ -206,15 +206,13 @@ public class ExpressionEstimation extends ExpressionVisitor<ColumnStatistic, Sta
return ColumnStatistic.UNKNOWN;
}
double literalVal = literal.getDouble();
- return new ColumnStatisticBuilder()
- .setMaxValue(literalVal)
- .setMinValue(literalVal)
- .setNdv(1)
- .setNumNulls(1)
- .setAvgSizeByte(1)
- .setMinExpr(literal.toLegacyLiteral())
- .setMaxExpr(literal.toLegacyLiteral())
- .build();
+ ColumnStatisticBuilder columnStatBuilder = new ColumnStatisticBuilder();
+ columnStatBuilder.setMaxValue(literalVal);
+ columnStatBuilder.setMinValue(literalVal);
+ columnStatBuilder.setNdv(1);
+ columnStatBuilder.setNumNulls(1);
+ columnStatBuilder.setAvgSizeByte(1);
+ return columnStatBuilder.build();
}
@Override
@@ -243,13 +241,13 @@ public class ExpressionEstimation extends ExpressionVisitor<ColumnStatistic, Sta
if (binaryArithmetic instanceof Add) {
return new ColumnStatisticBuilder().setCount(rowCount).setNdv(ndv).setAvgSizeByte(leftColStats.avgSizeByte)
.setNumNulls(numNulls).setDataSize(dataSize).setMinValue(leftMin + rightMin)
- .setMaxValue(leftMax + rightMax)
+ .setMaxValue(leftMax + rightMax).setSelectivity(1.0)
.setMinExpr(null).setMaxExpr(null).build();
}
if (binaryArithmetic instanceof Subtract) {
return new ColumnStatisticBuilder().setCount(rowCount).setNdv(ndv).setAvgSizeByte(leftColStats.avgSizeByte)
.setNumNulls(numNulls).setDataSize(dataSize).setMinValue(leftMin - rightMax)
- .setMaxValue(leftMax - rightMin).setMinExpr(null)
+ .setMaxValue(leftMax - rightMin).setSelectivity(1.0).setMinExpr(null)
.setMaxExpr(null).build();
}
// TODO: stat for multiply and divide produced by below algorithm may have huge deviation with reality.
@@ -261,11 +259,11 @@ public class ExpressionEstimation extends ExpressionVisitor<ColumnStatistic, Sta
leftMax * rightMax);
double max = Math.max(
Math.max(
- Math.max(leftMin * rightMin, leftMin * rightMax),
- leftMax * rightMin),
+ Math.max(leftMin * rightMin, leftMin * rightMax),
+ leftMax * rightMin),
leftMax * rightMax);
return new ColumnStatisticBuilder().setCount(rowCount).setNdv(ndv).setAvgSizeByte(leftColStats.avgSizeByte)
- .setNumNulls(numNulls).setDataSize(dataSize).setMinValue(min).setMaxValue(max)
+ .setNumNulls(numNulls).setDataSize(dataSize).setMinValue(min).setMaxValue(max).setSelectivity(1.0)
.setMaxExpr(null).setMinExpr(null).build();
}
if (binaryArithmetic instanceof Divide || binaryArithmetic instanceof IntegralDivide) {
@@ -281,7 +279,7 @@ public class ExpressionEstimation extends ExpressionVisitor<ColumnStatistic, Sta
leftMax / noneZeroDivisor(rightMax));
return new ColumnStatisticBuilder().setCount(rowCount).setNdv(ndv).setAvgSizeByte(leftColStats.avgSizeByte)
.setNumNulls(numNulls).setDataSize(binaryArithmetic.getDataType().width()).setMinValue(min)
- .setMaxValue(max).build();
+ .setMaxValue(max).setSelectivity(1.0).build();
}
if (binaryArithmetic instanceof Mod) {
double min = -Math.max(Math.abs(rightMin), Math.abs(rightMax));
@@ -311,12 +309,13 @@ public class ExpressionEstimation extends ExpressionVisitor<ColumnStatistic, Sta
}
/*
we keep columnStat.min and columnStat.max, but set ndv=1.
- if there is group-by keys, we will update count when visiting group clause
+ if there is group-by keys, we will update ndv when visiting group clause
*/
double width = min.child().getDataType().width();
- return new ColumnStatisticBuilder().setCount(1).setNdv(1).setAvgSizeByte(width)
- .setMinValue(columnStat.minValue).setMinExpr(columnStat.minExpr)
- .setMaxValue(columnStat.maxValue).setMaxExpr(columnStat.maxExpr).build();
+ return new ColumnStatisticBuilder().setCount(1).setNdv(1).setAvgSizeByte(width).setNumNulls(width)
+ .setDataSize(child.getDataType().width()).setMinValue(columnStat.minValue)
+ .setMaxValue(columnStat.maxValue).setSelectivity(1.0)
+ .setMinExpr(null).build();
}
@Override
@@ -328,20 +327,19 @@ public class ExpressionEstimation extends ExpressionVisitor<ColumnStatistic, Sta
}
/*
we keep columnStat.min and columnStat.max, but set ndv=1.
- if there is group-by keys, we will update count when visiting group clause
+ if there is group-by keys, we will update ndv when visiting group clause
*/
int width = max.child().getDataType().width();
- return new ColumnStatisticBuilder().setCount(1D).setNdv(1D).setAvgSizeByte(width)
- .setMinValue(columnStat.minValue).setMinExpr(columnStat.minExpr)
- .setMaxValue(columnStat.maxValue).setMaxExpr(columnStat.maxExpr)
- .build();
+ return new ColumnStatisticBuilder().setCount(1D).setNdv(1D).setAvgSizeByte(width).setNumNulls(0)
+ .setDataSize(width).setMinValue(columnStat.minValue).setMaxValue(columnStat.maxValue)
+ .setSelectivity(1.0).setMaxExpr(null).setMinExpr(null).build();
}
@Override
public ColumnStatistic visitCount(Count count, Statistics context) {
double width = count.getDataType().width();
return new ColumnStatisticBuilder().setCount(1D).setAvgSizeByte(width).setNumNulls(0)
- .setDataSize(width).setMinValue(0).setMaxValue(context.getRowCount())
+ .setDataSize(width).setMinValue(0).setMaxValue(context.getRowCount()).setSelectivity(1.0)
.setMaxExpr(null).setMinExpr(null).build();
}
@@ -369,7 +367,7 @@ public class ExpressionEstimation extends ExpressionVisitor<ColumnStatistic, Sta
.setNumNulls(childStat.numNulls)
.setDataSize(4 * childStat.count)
.setMinValue(minYear)
- .setMaxValue(maxYear).setMinExpr(null).build();
+ .setMaxValue(maxYear).setSelectivity(1.0).setMinExpr(null).build();
}
@Override
@@ -380,7 +378,7 @@ public class ExpressionEstimation extends ExpressionVisitor<ColumnStatistic, Sta
.setNdv(54)
.setAvgSizeByte(width)
.setNumNulls(childStat.numNulls)
- .setDataSize(1).setMinValue(1).setMaxValue(53).setMinExpr(null)
+ .setDataSize(1).setMinValue(1).setMaxValue(53).setSelectivity(1.0).setMinExpr(null)
.build();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
index f06c9d1cc4f..c5ddbd285b3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
@@ -17,10 +17,10 @@
package org.apache.doris.nereids.stats;
-import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.nereids.stats.FilterEstimation.EstimationContext;
import org.apache.doris.nereids.trees.TreeNode;
import org.apache.doris.nereids.trees.expressions.And;
+import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
import org.apache.doris.nereids.trees.expressions.CompoundPredicate;
import org.apache.doris.nereids.trees.expressions.EqualTo;
@@ -28,7 +28,6 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.GreaterThan;
import org.apache.doris.nereids.trees.expressions.GreaterThanEqual;
import org.apache.doris.nereids.trees.expressions.InPredicate;
-import org.apache.doris.nereids.trees.expressions.IsNull;
import org.apache.doris.nereids.trees.expressions.LessThan;
import org.apache.doris.nereids.trees.expressions.LessThanEqual;
import org.apache.doris.nereids.trees.expressions.Like;
@@ -38,7 +37,6 @@ import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.functions.Function;
-import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.statistics.Bucket;
import org.apache.doris.statistics.ColumnStatistic;
@@ -49,11 +47,10 @@ import org.apache.doris.statistics.StatisticRange;
import org.apache.doris.statistics.Statistics;
import org.apache.doris.statistics.StatisticsBuilder;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Sets;
-
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.function.Predicate;
@@ -84,9 +81,7 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
public Statistics estimate(Expression expression, Statistics statistics) {
// For a comparison predicate, only when it's left side is a slot and right side is a literal, we would
// consider is a valid predicate.
- Statistics stats = expression.accept(this, new EstimationContext(statistics));
- stats.enforceValid();
- return stats;
+ return expression.accept(this, new EstimationContext(statistics));
}
@Override
@@ -99,7 +94,7 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
Expression leftExpr = predicate.child(0);
Expression rightExpr = predicate.child(1);
Statistics leftStats = leftExpr.accept(this, context);
- Statistics andStats = rightExpr.accept(this,
+ Statistics andStats = rightExpr.accept(new FilterEstimation(),
new EstimationContext(leftStats));
if (predicate instanceof And) {
return andStats;
@@ -107,29 +102,27 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
Statistics rightStats = rightExpr.accept(this, context);
double rowCount = leftStats.getRowCount() + rightStats.getRowCount() - andStats.getRowCount();
Statistics orStats = context.statistics.withRowCount(rowCount);
- Set<Slot> leftInputSlots = leftExpr.getInputSlots();
- Set<Slot> rightInputSlots = rightExpr.getInputSlots();
- for (Slot slot : context.keyColumns) {
- if (leftInputSlots.contains(slot) && rightInputSlots.contains(slot)) {
- ColumnStatistic leftColStats = leftStats.findColumnStatistics(slot);
- ColumnStatistic rightColStats = rightStats.findColumnStatistics(slot);
- StatisticRange leftRange = StatisticRange.from(leftColStats, slot.getDataType());
- StatisticRange rightRange = StatisticRange.from(rightColStats, slot.getDataType());
- StatisticRange union = leftRange.union(rightRange);
- ColumnStatisticBuilder colBuilder = new ColumnStatisticBuilder(
- context.statistics.findColumnStatistics(slot));
- colBuilder.setMinValue(union.getLow()).setMinExpr(union.getLowExpr())
- .setMaxValue(union.getHigh()).setMaxExpr(union.getHighExpr())
- .setNdv(union.getDistinctValues());
- orStats.addColumnStats(slot, colBuilder.build());
+ for (Map.Entry<Expression, ColumnStatistic> entry : orStats.columnStatistics().entrySet()) {
+ ColumnStatistic leftColStats = leftStats.findColumnStatistics(entry.getKey());
+ ColumnStatistic rightColStats = rightStats.findColumnStatistics(entry.getKey());
+ ColumnStatisticBuilder estimatedColStatsBuilder = new ColumnStatisticBuilder(entry.getValue());
+ if (leftColStats.minValue <= rightColStats.minValue) {
+ estimatedColStatsBuilder.setMinValue(leftColStats.minValue);
+ estimatedColStatsBuilder.setMinExpr(leftColStats.minExpr);
+ } else {
+ estimatedColStatsBuilder.setMinValue(rightColStats.minValue);
+ estimatedColStatsBuilder.setMinExpr(rightColStats.minExpr);
+ }
+ if (leftColStats.maxValue >= rightColStats.maxValue) {
+ estimatedColStatsBuilder.setMaxValue(leftColStats.maxValue);
+ estimatedColStatsBuilder.setMaxExpr(leftColStats.maxExpr);
+ } else {
+ estimatedColStatsBuilder.setMaxValue(rightColStats.maxValue);
+ estimatedColStatsBuilder.setMaxExpr(rightColStats.maxExpr);
}
}
return orStats;
}
- // should not come here
- Preconditions.checkArgument(false,
- "unsupported compound operator: %s in %s",
- predicate.getClass().getName(), predicate.toSql());
return context.statistics;
}
@@ -179,27 +172,25 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
}
private Statistics updateLessThanLiteral(Expression leftExpr, ColumnStatistic statsForLeft,
- ColumnStatistic statsForRight, EstimationContext context, boolean contains) {
+ double val, EstimationContext context, boolean contains) {
if (statsForLeft.hasHistogram()) {
- return estimateLessThanLiteralWithHistogram(leftExpr, statsForLeft,
- statsForRight.maxValue, context, contains);
+ return estimateLessThanLiteralWithHistogram(leftExpr, statsForLeft, val, context, contains);
}
- StatisticRange rightRange = new StatisticRange(statsForLeft.minValue, statsForLeft.minExpr,
- statsForRight.maxValue, statsForRight.maxExpr,
- statsForLeft.ndv, leftExpr.getDataType());
+ //rightRange.distinctValues should not be used
+ StatisticRange rightRange = new StatisticRange(statsForLeft.minValue, val, statsForLeft.ndv,
+ leftExpr.getDataType());
return estimateBinaryComparisonFilter(leftExpr,
statsForLeft,
rightRange, context);
}
private Statistics updateGreaterThanLiteral(Expression leftExpr, ColumnStatistic statsForLeft,
- ColumnStatistic statsForRight, EstimationContext context, boolean contains) {
+ double val, EstimationContext context, boolean contains) {
if (statsForLeft.hasHistogram()) {
- return estimateGreaterThanLiteralWithHistogram(leftExpr, statsForLeft,
- statsForRight.minValue, context, contains);
+ return estimateGreaterThanLiteralWithHistogram(leftExpr, statsForLeft, val, context, contains);
}
- StatisticRange rightRange = new StatisticRange(statsForRight.minValue, statsForRight.minExpr,
- statsForLeft.maxValue, statsForLeft.maxExpr,
+ //rightRange.distinctValues should not be used
+ StatisticRange rightRange = new StatisticRange(val, statsForLeft.maxValue,
statsForLeft.ndv, leftExpr.getDataType());
return estimateBinaryComparisonFilter(leftExpr, statsForLeft, rightRange, context);
}
@@ -213,12 +204,12 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
if (cp instanceof EqualTo || cp instanceof NullSafeEqual) {
return estimateEqualTo(cp, statsForLeft, statsForRight, context);
} else {
+ double val = statsForRight.maxValue;
if (cp instanceof LessThan || cp instanceof LessThanEqual) {
- return updateLessThanLiteral(cp.left(), statsForLeft, statsForRight,
- context, cp instanceof LessThanEqual);
+ return updateLessThanLiteral(cp.left(), statsForLeft, val, context, cp instanceof LessThanEqual);
} else if (cp instanceof GreaterThan || cp instanceof GreaterThanEqual) {
- return updateGreaterThanLiteral(cp.left(), statsForLeft, statsForRight, context,
+ return updateGreaterThanLiteral(cp.left(), statsForLeft, val, context,
cp instanceof GreaterThanEqual);
} else {
throw new RuntimeException(String.format("Unexpected expression : %s", cp.toSql()));
@@ -243,10 +234,19 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
Statistics equalStats = context.statistics.withSel(selectivity);
Expression left = cp.left();
- equalStats.addColumnStats(left, statsForRight);
- context.addKeyIfSlot(left);
- if (!(left instanceof SlotReference)) {
- left.accept(new ColumnStatsAdjustVisitor(), equalStats);
+ if (left instanceof Cast) {
+ left = ((Cast) left).child();
+ }
+ if (left instanceof SlotReference) {
+ Slot leftSlot = (SlotReference) left;
+ //update min/max of cp.left
+ ColumnStatistic columnStats = equalStats.findColumnStatistics(leftSlot);
+ ColumnStatisticBuilder colStatsBuilder = new ColumnStatisticBuilder(columnStats);
+ colStatsBuilder.setMaxValue(val);
+ colStatsBuilder.setMinValue(val);
+ colStatsBuilder.setNdv(1);
+ colStatsBuilder.setNumNulls(0);
+ equalStats.addColumnStats(leftSlot, colStatsBuilder.build());
}
return equalStats;
}
@@ -275,12 +275,8 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
return context.statistics.withSel(DEFAULT_IN_COEFFICIENT);
}
List<Expression> options = inPredicate.getOptions();
- // init minOption and maxOption by compareExpr.max and compareExpr.min respectively,
- // and then adjust min/max by options
- double minOptionValue = compareExprStats.maxValue;
- double maxOptionValue = compareExprStats.minValue;
- LiteralExpr minOptionLiteral = compareExprStats.maxExpr;
- LiteralExpr maxOptionLiteral = compareExprStats.minExpr;
+ double maxOption = 0;
+ double minOption = Double.MAX_VALUE;
/* suppose A.(min, max) = (0, 10), A.ndv=10
A in ( 1, 2, 5, 100):
validInOptCount = 3, that is (1, 2, 5)
@@ -296,199 +292,86 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
A.(min, max) not changed
A.selectivity = 7/10
*/
- int validInOptCount = 0;
+ double validInOptCount = 0;
double selectivity = 1.0;
ColumnStatisticBuilder compareExprStatsBuilder = new ColumnStatisticBuilder(compareExprStats);
- int nonLiteralOptionCount = 0;
+
for (Expression option : options) {
ColumnStatistic optionStats = ExpressionEstimation.estimate(option, context.statistics);
- if (option instanceof Literal) {
- // remove the options which is out of compareExpr.range
- if (compareExprStats.minValue <= optionStats.maxValue
- && optionStats.maxValue <= compareExprStats.maxValue) {
- validInOptCount++;
- LiteralExpr optionLiteralExpr = ((Literal) option).toLegacyLiteral();
- if (maxOptionLiteral == null || optionLiteralExpr.compareTo(maxOptionLiteral) >= 0) {
- maxOptionLiteral = optionLiteralExpr;
- maxOptionValue = optionStats.maxValue;
- }
-
- if (minOptionLiteral == null || optionLiteralExpr.compareTo(minOptionLiteral) <= 0) {
- minOptionLiteral = optionLiteralExpr;
- minOptionValue = optionStats.minValue;
- }
- }
- } else {
- nonLiteralOptionCount++;
- }
- }
- if (nonLiteralOptionCount > 0) {
- // A in (x+1, ...)
- // "x+1" is not literal, and if const-fold can not handle it, it blocks estimation of min/max value.
- // and hence, we do not adjust compareExpr.stats.range.
- int newNdv = nonLiteralOptionCount + validInOptCount;
- if (newNdv < compareExprStats.ndv) {
- compareExprStatsBuilder.setNdv(newNdv);
- selectivity = StatsMathUtil.divide(newNdv, compareExprStats.ndv);
- } else {
- selectivity = 1.0;
- }
- } else {
- maxOptionValue = Math.min(maxOptionValue, compareExprStats.maxValue);
- minOptionValue = Math.max(minOptionValue, compareExprStats.minValue);
- compareExprStatsBuilder.setMaxValue(maxOptionValue);
- compareExprStatsBuilder.setMaxExpr(maxOptionLiteral);
- compareExprStatsBuilder.setMinValue(minOptionValue);
- compareExprStatsBuilder.setMinExpr(minOptionLiteral);
- if (validInOptCount < compareExprStats.ndv) {
- compareExprStatsBuilder.setNdv(validInOptCount);
- selectivity = StatsMathUtil.divide(validInOptCount, compareExprStats.ndv);
- } else {
- selectivity = 1.0;
+ double validOptionNdv = compareExprStats.ndvIntersection(optionStats);
+ if (validOptionNdv > 0.0) {
+ validInOptCount += validOptionNdv;
+ maxOption = Math.max(optionStats.maxValue, maxOption);
+ minOption = Math.min(optionStats.minValue, minOption);
}
}
+ maxOption = Math.min(maxOption, compareExprStats.maxValue);
+ minOption = Math.max(minOption, compareExprStats.minValue);
+ compareExprStatsBuilder.setMaxValue(maxOption);
+ compareExprStatsBuilder.setMinValue(minOption);
+
+ selectivity = StatsMathUtil.minNonNaN(1.0, validInOptCount / compareExprStats.ndv);
+ compareExprStatsBuilder.setNdv(validInOptCount);
Statistics estimated = new Statistics(context.statistics);
estimated = estimated.withSel(selectivity);
- estimated.addColumnStats(compareExpr,
- compareExprStatsBuilder.build());
- context.addKeyIfSlot(compareExpr);
+ if (compareExpr instanceof SlotReference) {
+ estimated.addColumnStats(compareExpr,
+ compareExprStatsBuilder.build());
+ }
return estimated;
}
- // Right Now, we just assume the selectivity is 1 when stats is Unknown
- private Statistics handleUnknownCase(EstimationContext context) {
- return context.statistics;
- }
-
@Override
public Statistics visitNot(Not not, EstimationContext context) {
- if (context.statistics.isInputSlotsUnknown(not.getInputSlots())) {
- return handleUnknownCase(context);
- }
- Expression child = not.child();
- Statistics childStats = child.accept(this, context);
+ Statistics childStats = new FilterEstimation().estimate(not.child(), context.statistics);
//if estimated rowCount is 0, adjust to 1 to make upper join reorder reasonable.
double rowCount = Math.max(context.statistics.getRowCount() - childStats.getRowCount(), 1);
StatisticsBuilder statisticsBuilder = new StatisticsBuilder(context.statistics).setRowCount(rowCount);
- // update key col stats
- for (Slot slot : not.child().getInputSlots()) {
- ColumnStatistic originColStats = context.statistics.findColumnStatistics(slot);
- ColumnStatistic childColStats = childStats.findColumnStatistics(slot);
- if (context.isKeySlot(slot)) {
- ColumnStatisticBuilder colBuilder = new ColumnStatisticBuilder(childColStats);
- // update column stats for
- // 1. not (A=B)
- // 2. not A in (...)
- // 3. not A is null
- // 4. not A like XXX
- colBuilder.setNumNulls(0);
- Preconditions.checkArgument(
- child instanceof EqualTo
- || child instanceof InPredicate
- || child instanceof IsNull
- || child instanceof Like,
- "Not-predicate meet unexpected child: %s", child.toSql());
- if (child instanceof Like) {
- rowCount = context.statistics.getRowCount() - childStats.getRowCount();
- colBuilder.setNdv(originColStats.ndv - childColStats.ndv);
- } else if (child instanceof InPredicate) {
- colBuilder.setNdv(originColStats.ndv - childColStats.ndv);
- colBuilder.setMinValue(originColStats.minValue)
- .setMinExpr(originColStats.minExpr)
- .setMaxValue(originColStats.maxValue)
- .setMaxExpr(originColStats.maxExpr);
- } else if (child instanceof IsNull) {
- colBuilder.setNdv(originColStats.ndv);
- colBuilder.setMinValue(originColStats.minValue)
- .setMinExpr(originColStats.minExpr)
- .setMaxValue(originColStats.maxValue)
- .setMaxExpr(originColStats.maxExpr);
- } else if (child instanceof EqualTo) {
- colBuilder.setNdv(originColStats.ndv - childColStats.ndv);
- colBuilder.setMinValue(originColStats.minValue)
- .setMinExpr(originColStats.minExpr)
- .setMaxValue(originColStats.maxValue)
- .setMaxExpr(originColStats.maxExpr);
- }
- statisticsBuilder.putColumnStatistics(slot, colBuilder.build());
+ for (Entry<Expression, ColumnStatistic> entry : context.statistics.columnStatistics().entrySet()) {
+ Expression expr = entry.getKey();
+ ColumnStatistic originColStats = entry.getValue();
+ ColumnStatistic childColStats = childStats.findColumnStatistics(expr);
+ double originNonNullCount = Math.max(originColStats.count - originColStats.numNulls, 0);
+ double childNonNullCount = Math.max(childColStats.count - childColStats.numNulls, 0);
+ double supersetValuesPerDistinctValue = StatsMathUtil.divide(originNonNullCount, originColStats.ndv);
+ double subsetValuesPerDistinctValue = StatsMathUtil.divide(childNonNullCount, childColStats.ndv);
+ double ndv;
+ if (supersetValuesPerDistinctValue <= subsetValuesPerDistinctValue) {
+ ndv = Math.max(originColStats.ndv - childColStats.ndv, 0);
+ } else {
+ ndv = originColStats.ndv;
}
+ double nullCount = Math.max(originColStats.numNulls - childColStats.numNulls, 0);
+ ColumnStatistic columnStatistic = new ColumnStatisticBuilder(originColStats)
+ .setNdv(ndv)
+ .setNumNulls(nullCount)
+ .build();
+ statisticsBuilder.putColumnStatistics(expr, columnStatistic);
}
-
return statisticsBuilder.build();
}
- @Override
- public Statistics visitIsNull(IsNull isNull, EstimationContext context) {
- ColumnStatistic childStats = ExpressionEstimation.estimate(isNull.child(), context.statistics);
- if (childStats.isUnKnown()) {
- return new StatisticsBuilder(context.statistics).build();
- }
- double outputRowCount = childStats.numNulls;
- ColumnStatisticBuilder colBuilder = new ColumnStatisticBuilder(childStats);
- colBuilder.setCount(outputRowCount).setNumNulls(outputRowCount)
- .setMaxValue(Double.POSITIVE_INFINITY)
- .setMinValue(Double.NEGATIVE_INFINITY)
- .setNdv(0);
- StatisticsBuilder builder = new StatisticsBuilder(context.statistics);
- builder.putColumnStatistics(isNull.child(), colBuilder.build());
- context.addKeyIfSlot(isNull.child());
- return builder.build();
- }
-
static class EstimationContext {
private final Statistics statistics;
- private final Set<Slot> keyColumns = Sets.newHashSet();
-
public EstimationContext(Statistics statistics) {
this.statistics = statistics;
}
-
- public void addKeyIfSlot(Expression expr) {
- if (expr instanceof Slot) {
- keyColumns.add((Slot) expr);
- }
- }
-
- public boolean isKeySlot(Expression expr) {
- if (expr instanceof Slot) {
- return keyColumns.contains((Slot) expr);
- }
- return false;
- }
}
private Statistics estimateBinaryComparisonFilter(Expression leftExpr, ColumnStatistic leftStats,
StatisticRange rightRange, EstimationContext context) {
StatisticRange leftRange =
- new StatisticRange(leftStats.minValue, leftStats.minExpr, leftStats.maxValue, leftStats.maxExpr,
- leftStats.ndv, leftExpr.getDataType());
+ new StatisticRange(leftStats.minValue, leftStats.maxValue, leftStats.ndv, leftExpr.getDataType());
StatisticRange intersectRange = leftRange.cover(rightRange);
-
- ColumnStatisticBuilder leftColumnStatisticBuilder;
- Statistics updatedStatistics;
- if (intersectRange.isEmpty()) {
- updatedStatistics = context.statistics.withRowCount(0);
- leftColumnStatisticBuilder = new ColumnStatisticBuilder(leftStats)
- .setMinValue(Double.NEGATIVE_INFINITY)
- .setMinExpr(null)
- .setMaxValue(Double.POSITIVE_INFINITY)
- .setMaxExpr(null)
- .setNdv(0)
- .setCount(0);
- } else {
- leftColumnStatisticBuilder = new ColumnStatisticBuilder(leftStats)
- .setMinValue(intersectRange.getLow())
- .setMinExpr(intersectRange.getLowExpr())
- .setMaxValue(intersectRange.getHigh())
- .setMaxExpr(intersectRange.getHighExpr())
- .setNdv(intersectRange.getDistinctValues());
- double sel = leftRange.overlapPercentWith(rightRange);
- updatedStatistics = context.statistics.withSel(sel);
- leftColumnStatisticBuilder.setCount(updatedStatistics.getRowCount());
- }
+ ColumnStatisticBuilder leftColumnStatisticBuilder = new ColumnStatisticBuilder(leftStats)
+ .setMinValue(intersectRange.getLow())
+ .setMaxValue(intersectRange.getHigh())
+ .setNdv(intersectRange.getDistinctValues());
+ double sel = leftRange.overlapPercentWith(rightRange);
+ Statistics updatedStatistics = context.statistics.withSel(sel);
+ leftColumnStatisticBuilder.setCount(updatedStatistics.getRowCount());
updatedStatistics.addColumnStats(leftExpr, leftColumnStatisticBuilder.build());
- context.addKeyIfSlot(leftExpr);
leftExpr.accept(new ColumnStatsAdjustVisitor(), updatedStatistics);
return updatedStatistics;
}
@@ -498,17 +381,36 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
StatisticRange leftRange = StatisticRange.from(leftStats, leftExpr.getDataType());
StatisticRange rightRange = StatisticRange.from(rightStats, rightExpr.getDataType());
StatisticRange leftIntersectRight = leftRange.intersect(rightRange);
- StatisticRange intersect = rightRange.intersect(leftIntersectRight);
- ColumnStatisticBuilder intersectBuilder = new ColumnStatisticBuilder(leftStats);
- intersectBuilder.setNdv(intersect.getDistinctValues());
- intersectBuilder.setMinValue(intersect.getLow());
- intersectBuilder.setMaxValue(intersect.getHigh());
- double sel = 1 / StatsMathUtil.nonZeroDivisor(Math.max(leftStats.ndv, rightStats.ndv));
+ StatisticRange rightIntersectLeft = rightRange.intersect(leftIntersectRight);
+ ColumnStatisticBuilder leftBuilder = new ColumnStatisticBuilder(leftStats);
+ leftBuilder.setNdv(leftIntersectRight.getDistinctValues());
+ leftBuilder.setMinValue(leftIntersectRight.getLow());
+ leftBuilder.setMaxValue(leftIntersectRight.getHigh());
+ ColumnStatisticBuilder rightBuilder = new ColumnStatisticBuilder(rightStats);
+ rightBuilder.setNdv(rightIntersectLeft.getDistinctValues());
+ rightBuilder.setMinValue(rightIntersectLeft.getLow());
+ rightBuilder.setMaxValue(rightIntersectLeft.getDistinctValues());
+ double sel;
+ double reduceRatio = 0.25;
+ double bothSideReducedRatio = 0.9;
+ if (!leftStats.rangeChanged() && !rightStats.rangeChanged()
+ && leftStats.ndv < leftStats.getOriginalNdv() * bothSideReducedRatio
+ && rightStats.ndv < rightStats.getOriginalNdv() * bothSideReducedRatio) {
+ double sel1;
+ if (leftStats.ndv > rightStats.ndv) {
+ sel1 = 1 / StatsMathUtil.nonZeroDivisor(leftStats.ndv);
+ } else {
+ sel1 = 1 / StatsMathUtil.nonZeroDivisor(rightStats.ndv);
+ }
+ double sel2 = Math.min(rightStats.ndv / rightStats.getOriginalNdv(),
+ leftStats.ndv / leftStats.getOriginalNdv());
+ sel = sel1 * Math.pow(sel2, reduceRatio);
+ } else {
+ sel = 1 / StatsMathUtil.nonZeroDivisor(Math.max(leftStats.ndv, rightStats.ndv));
+ }
Statistics updatedStatistics = context.statistics.withSel(sel);
- updatedStatistics.addColumnStats(leftExpr, intersectBuilder.build());
- updatedStatistics.addColumnStats(rightExpr, intersectBuilder.build());
- context.addKeyIfSlot(leftExpr);
- context.addKeyIfSlot(rightExpr);
+ updatedStatistics.addColumnStats(leftExpr, leftBuilder.build());
+ updatedStatistics.addColumnStats(rightExpr, rightBuilder.build());
return updatedStatistics;
}
@@ -524,8 +426,6 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
context.statistics.getRowCount() - rightStats.numNulls));
statistics.addColumnStats(leftExpr, new ColumnStatisticBuilder(leftStats).setNumNulls(0.0).build());
statistics.addColumnStats(rightExpr, new ColumnStatisticBuilder(rightStats).setNumNulls(0.0).build());
- context.addKeyIfSlot(leftExpr);
- context.addKeyIfSlot(rightExpr);
return statistics;
}
double leftOverlapPercent = leftRange.overlapPercentWith(rightRange);
@@ -533,8 +433,8 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
if (leftOverlapPercent == 0) {
return context.statistics.withRowCount(0.0);
}
- StatisticRange leftAlwaysLessThanRightRange = new StatisticRange(leftStats.minValue, leftStats.minExpr,
- rightStats.minValue, rightStats.minExpr, Double.NaN, leftExpr.getDataType());
+ StatisticRange leftAlwaysLessThanRightRange = new StatisticRange(leftStats.minValue,
+ rightStats.minValue, Double.NaN, leftExpr.getDataType());
double leftAlwaysLessThanRightPercent = 0;
if (leftRange.getLow() < rightRange.getLow()) {
leftAlwaysLessThanRightPercent = leftRange.overlapPercentWith(leftAlwaysLessThanRightRange);
@@ -548,10 +448,8 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
double rightOverlappingRangeFraction = rightRange.overlapPercentWith(leftRange);
double rightAlwaysGreaterRangeFraction = 0;
if (leftRange.getHigh() < rightRange.getHigh()) {
- rightAlwaysGreaterRangeFraction = rightRange.overlapPercentWith(new StatisticRange(
- leftRange.getHigh(), leftRange.getHighExpr(),
- rightRange.getHigh(), rightRange.getHighExpr(),
- Double.NaN, rightExpr.getDataType()));
+ rightAlwaysGreaterRangeFraction = rightRange.overlapPercentWith(new StatisticRange(leftRange.getHigh(),
+ rightRange.getHigh(), Double.NaN, rightExpr.getDataType()));
}
ColumnStatistic rightColumnStatistic = new ColumnStatisticBuilder(rightStats)
.setMinValue(Math.max(leftRange.getLow(), rightRange.getLow()))
@@ -562,8 +460,6 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
double sel = leftAlwaysLessThanRightPercent
+ leftOverlapPercent * rightOverlappingRangeFraction * DEFAULT_INEQUALITY_COEFFICIENT
+ leftOverlapPercent * rightAlwaysGreaterRangeFraction;
- context.addKeyIfSlot(leftExpr);
- context.addKeyIfSlot(rightExpr);
return context.statistics.withSel(sel)
.addColumnStats(leftExpr, leftColumnStatistic)
.addColumnStats(rightExpr, rightColumnStatistic);
@@ -597,7 +493,6 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
.setMaxValue(numVal)
.setHistogram(new HistogramBuilder(leftHist).setBuckets(updatedBucketList).build())
.build();
- context.addKeyIfSlot(leftExpr);
return context.statistics.withSel(sel).addColumnStats(leftExpr, columnStatistic);
}
}
@@ -625,7 +520,7 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
double overlapCountInBucket = overlapPercentInBucket * bucket.count;
double sel = StatsMathUtil.minNonNaN(1,
(leftHist.size() - bucket.preSum - (bucket.count - overlapCountInBucket))
- / context.statistics.getRowCount());
+ / context.statistics.getRowCount());
List<Bucket> updatedBucketList = new ArrayList<>();
updatedBucketList.add(new Bucket(numVal, bucket.upper, overlapPercentInBucket * bucket.count,
0, overlapPercentInBucket * bucket.ndv));
@@ -634,7 +529,6 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
.setMaxValue(numVal)
.setHistogram(new HistogramBuilder(leftHist).setBuckets(updatedBucketList).build())
.build();
- context.addKeyIfSlot(leftExpr);
return context.statistics.withSel(sel).addColumnStats(leftExpr, columnStatistic);
}
}
@@ -662,24 +556,11 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
.setMaxValue(numVal)
.setMinValue(numVal)
.build();
- context.addKeyIfSlot(leftExpr);
return context.statistics.withSel(sel).addColumnStats(leftExpr, columnStatistic);
}
@Override
public Statistics visitLike(Like like, EstimationContext context) {
- StatisticsBuilder statsBuilder = new StatisticsBuilder(context.statistics);
- statsBuilder.setRowCount(context.statistics.getRowCount() * DEFAULT_LIKE_COMPARISON_SELECTIVITY);
- if (like.left() instanceof Slot) {
- ColumnStatistic origin = context.statistics.findColumnStatistics(like.left());
- Preconditions.checkArgument(origin != null,
- "col stats not found. slot=%s in %s",
- like.left().toSql(), like.toSql());
- ColumnStatisticBuilder colBuilder = new ColumnStatisticBuilder(origin);
- colBuilder.setNdv(origin.ndv * DEFAULT_LIKE_COMPARISON_SELECTIVITY).setNumNulls(0);
- statsBuilder.putColumnStatistics(like.left(), colBuilder.build());
- context.addKeyIfSlot(like.left());
- }
- return statsBuilder.build();
+ return context.statistics.withSel(DEFAULT_LIKE_COMPARISON_SELECTIVITY);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java
index ef4575e3308..a40e409fae4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java
@@ -72,7 +72,16 @@ public class JoinEstimation {
return false;
}
- private static Statistics estimateHashJoin(Statistics leftStats, Statistics rightStats, Join join) {
+ private static Statistics estimateInnerJoin(Statistics leftStats, Statistics rightStats, Join join) {
+ if (hashJoinConditionContainsUnknownColumnStats(leftStats, rightStats, join)) {
+ double rowCount = Math.max(leftStats.getRowCount(), rightStats.getRowCount());
+ rowCount = Math.max(1, rowCount);
+ return new StatisticsBuilder()
+ .setRowCount(rowCount)
+ .putColumnStatistics(leftStats.columnStatistics())
+ .putColumnStatistics(rightStats.columnStatistics())
+ .build();
+ }
/*
* When we estimate filter A=B,
* if any side of equation, A or B, is almost unique, the confidence level of estimation is high.
@@ -120,7 +129,7 @@ public class JoinEstimation {
.putColumnStatistics(rightStats.columnStatistics())
.build();
- double outputRowCount;
+ double outputRowCount = 1;
if (!trustableConditions.isEmpty()) {
List<Double> joinConditionSels = trustableConditions.stream()
.map(expression -> estimateJoinConditionSel(crossJoinStats, expression))
@@ -129,47 +138,20 @@ public class JoinEstimation {
double sel = 1.0;
double denominator = 1.0;
- for (Double joinConditionSel : joinConditionSels) {
- sel *= Math.pow(joinConditionSel, 1 / denominator);
+ for (int i = 0; i < joinConditionSels.size(); i++) {
+ sel *= Math.pow(joinConditionSels.get(i), 1 / denominator);
denominator *= 2;
}
outputRowCount = Math.max(1, crossJoinStats.getRowCount() * sel);
outputRowCount = outputRowCount * Math.pow(0.9, unTrustableCondition.size());
+ innerJoinStats = crossJoinStats.updateRowCountOnly(outputRowCount);
} else {
outputRowCount = Math.max(leftStats.getRowCount(), rightStats.getRowCount());
Optional<Double> ratio = unTrustEqualRatio.stream().min(Double::compareTo);
if (ratio.isPresent()) {
outputRowCount = Math.max(1, outputRowCount * ratio.get());
}
- }
- innerJoinStats = crossJoinStats.withRowCountAndEnforceValid(outputRowCount);
- return innerJoinStats;
- }
-
- private static Statistics estimateNestLoopJoin(Statistics leftStats, Statistics rightStats, Join join) {
- return new StatisticsBuilder()
- .setRowCount(Math.max(1, leftStats.getRowCount() * rightStats.getRowCount()))
- .putColumnStatistics(leftStats.columnStatistics())
- .putColumnStatistics(rightStats.columnStatistics())
- .build();
- }
-
- private static Statistics estimateInnerJoin(Statistics leftStats, Statistics rightStats, Join join) {
- if (hashJoinConditionContainsUnknownColumnStats(leftStats, rightStats, join)) {
- double rowCount = Math.max(leftStats.getRowCount(), rightStats.getRowCount());
- rowCount = Math.max(1, rowCount);
- return new StatisticsBuilder()
- .setRowCount(rowCount)
- .putColumnStatistics(leftStats.columnStatistics())
- .putColumnStatistics(rightStats.columnStatistics())
- .build();
- }
-
- Statistics innerJoinStats;
- if (join.getHashJoinConjuncts().isEmpty()) {
- innerJoinStats = estimateNestLoopJoin(leftStats, rightStats, join);
- } else {
- innerJoinStats = estimateHashJoin(leftStats, rightStats, join);
+ innerJoinStats = crossJoinStats.updateRowCountOnly(outputRowCount);
}
if (!join.getOtherJoinConjuncts().isEmpty()) {
@@ -180,6 +162,9 @@ public class JoinEstimation {
innerJoinStats = new StatisticsBuilder(innerJoinStats).setRowCount(1).build();
}
}
+
+ innerJoinStats.setWidth(leftStats.getWidth() + rightStats.getWidth());
+ innerJoinStats.setPenalty(0);
return innerJoinStats;
}
@@ -257,9 +242,10 @@ public class JoinEstimation {
double baseRowCount =
join.getJoinType().isLeftSemiOrAntiJoin() ? leftStats.getRowCount() : rightStats.getRowCount();
rowCount = Math.min(innerJoinStats.getRowCount(), baseRowCount);
- return innerJoinStats.withRowCountAndEnforceValid(rowCount);
+ return innerJoinStats.withRowCount(rowCount);
} else {
StatisticsBuilder builder;
+ double originalRowCount = leftStats.getRowCount();
if (join.getJoinType().isLeftSemiOrAntiJoin()) {
builder = new StatisticsBuilder(leftStats);
builder.setRowCount(rowCount);
@@ -267,9 +253,10 @@ public class JoinEstimation {
//right semi or anti
builder = new StatisticsBuilder(rightStats);
builder.setRowCount(rowCount);
+ originalRowCount = rightStats.getRowCount();
}
Statistics outputStats = builder.build();
- outputStats.enforceValid();
+ outputStats.fix(rowCount, originalRowCount);
return outputStats;
}
}
@@ -289,15 +276,15 @@ public class JoinEstimation {
Statistics innerJoinStats = estimateInnerJoin(leftStats, rightStats, join);
double rowCount = Math.max(leftStats.getRowCount(), innerJoinStats.getRowCount());
rowCount = Math.max(leftStats.getRowCount(), rowCount);
- return innerJoinStats.withRowCountAndEnforceValid(rowCount);
+ return innerJoinStats.withRowCount(rowCount);
} else if (joinType == JoinType.RIGHT_OUTER_JOIN) {
Statistics innerJoinStats = estimateInnerJoin(leftStats, rightStats, join);
double rowCount = Math.max(rightStats.getRowCount(), innerJoinStats.getRowCount());
rowCount = Math.max(rowCount, rightStats.getRowCount());
- return innerJoinStats.withRowCountAndEnforceValid(rowCount);
+ return innerJoinStats.withRowCount(rowCount);
} else if (joinType == JoinType.FULL_OUTER_JOIN) {
Statistics innerJoinStats = estimateInnerJoin(leftStats, rightStats, join);
- return innerJoinStats.withRowCountAndEnforceValid(leftStats.getRowCount()
+ return innerJoinStats.withRowCount(leftStats.getRowCount()
+ rightStats.getRowCount() + innerJoinStats.getRowCount());
} else if (joinType == JoinType.CROSS_JOIN) {
return new StatisticsBuilder()
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index 48ea4c9800d..6910f5b4af3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -17,7 +17,10 @@
package org.apache.doris.nereids.stats;
+import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.SchemaTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.Config;
import org.apache.doris.common.FeConstants;
@@ -617,7 +620,7 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
double rowCount = catalogRelation.getTable().estimatedRowCount();
for (SlotReference slotReference : slotSet) {
String colName = slotReference.getName();
- boolean shouldIgnoreThisCol = StatisticConstants.shouldIgnoreCol(table, slotReference.getColumn().get());
+ boolean shouldIgnoreThisCol = shouldIgnoreCol(table, slotReference.getColumn().get());
if (colName == null) {
throw new RuntimeException(String.format("Invalid slot: %s", slotReference.getExprId()));
@@ -655,20 +658,16 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
}
private Statistics computePartitionTopN(PartitionTopN partitionTopN) {
- Statistics childStats = groupExpression.childStatistics(0);
- double rowCount = childStats.getRowCount();
+ Statistics stats = groupExpression.childStatistics(0);
+ double rowCount = stats.getRowCount();
List<Expression> partitionKeys = partitionTopN.getPartitionKeys();
if (!partitionTopN.hasGlobalLimit() && !partitionKeys.isEmpty()) {
// If there is no global limit. So result for the cardinality estimation is:
// NDV(partition key) * partitionLimit
+ Map<Expression, ColumnStatistic> childSlotToColumnStats = stats.columnStatistics();
List<ColumnStatistic> partitionByKeyStats = partitionKeys.stream()
- .map(partitionKey -> {
- ColumnStatistic partitionKeyStats = childStats.findColumnStatistics(partitionKey);
- if (partitionKeyStats == null) {
- partitionKeyStats = new ExpressionEstimation().visit(partitionKey, childStats);
- }
- return partitionKeyStats;
- })
+ .filter(childSlotToColumnStats::containsKey)
+ .map(childSlotToColumnStats::get)
.filter(s -> !s.isUnKnown)
.collect(Collectors.toList());
if (partitionByKeyStats.isEmpty()) {
@@ -676,7 +675,7 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
rowCount = rowCount * DEFAULT_COLUMN_NDV_RATIO;
} else {
rowCount = Math.min(rowCount, partitionByKeyStats.stream().map(s -> s.ndv)
- .max(Double::compare).get() * partitionTopN.getPartitionLimit());
+ .max(Double::compare).get());
}
} else {
rowCount = Math.min(rowCount, partitionTopN.getPartitionLimit());
@@ -684,7 +683,7 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
// TODO: for the filter push down window situation, we will prune the row count twice
// because we keep the pushed down filter. And it will be calculated twice, one of them in 'PartitionTopN'
// and the other is in 'Filter'. It's hard to dismiss.
- return childStats.withRowCountAndEnforceValid(rowCount);
+ return stats.updateRowCountOnly(rowCount);
}
private Statistics computeLimit(Limit limit) {
@@ -753,7 +752,9 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
builder.setDataSize(rowCount * outputExpression.getDataType().width());
slotToColumnStats.put(outputExpression.toSlot(), columnStat);
}
- return new Statistics(rowCount, slotToColumnStats);
+ return new Statistics(rowCount, slotToColumnStats, childStats.getWidth(),
+ childStats.getPenalty() + childStats.getRowCount());
+ // TODO: Update ColumnStats properly, add new mapping from output slot to ColumnStats
}
private Statistics computeRepeat(Repeat<? extends Plan> repeat) {
@@ -771,7 +772,8 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
.setDataSize(stats.dataSize < 0 ? stats.dataSize : stats.dataSize * groupingSetNum);
return Pair.of(kv.getKey(), columnStatisticBuilder.build());
}).collect(Collectors.toMap(Pair::key, Pair::value));
- return new Statistics(rowCount < 0 ? rowCount : rowCount * groupingSetNum, columnStatisticMap);
+ return new Statistics(rowCount < 0 ? rowCount : rowCount * groupingSetNum, columnStatisticMap,
+ childStats.getWidth(), childStats.getPenalty());
}
private Statistics computeProject(Project project) {
@@ -781,7 +783,7 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
ColumnStatistic columnStatistic = ExpressionEstimation.estimate(projection, childStats);
return new SimpleEntry<>(projection.toSlot(), columnStatistic);
}).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (item1, item2) -> item1));
- return new Statistics(childStats.getRowCount(), columnsStats);
+ return new Statistics(childStats.getRowCount(), columnsStats, childStats.getWidth(), childStats.getPenalty());
}
private Statistics computeOneRowRelation(List<NamedExpression> projects) {
@@ -1065,4 +1067,16 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
return groupExpression.childStatistics(1);
}
+ private boolean shouldIgnoreCol(TableIf tableIf, Column c) {
+ if (tableIf instanceof SchemaTable) {
+ return true;
+ }
+ if (tableIf instanceof OlapTable) {
+ OlapTable olapTable = (OlapTable) tableIf;
+ if (StatisticConstants.STATISTICS_DB_BLACK_LIST.contains(olapTable.getQualifiedDbName())) {
+ return true;
+ }
+ }
+ return !c.isVisible();
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Properties.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Properties.java
deleted file mode 100644
index db0c78c1f78..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Properties.java
+++ /dev/null
@@ -1,96 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.nereids.trees.expressions;
-
-import org.apache.doris.nereids.exceptions.UnboundException;
-import org.apache.doris.nereids.trees.expressions.shape.LeafExpression;
-import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
-import org.apache.doris.nereids.types.DataType;
-import org.apache.doris.nereids.types.MapType;
-
-import com.google.common.collect.ImmutableList;
-
-import java.util.Map;
-import java.util.Objects;
-import java.util.stream.Collectors;
-
-/**
- * Properties
- */
-public class Properties extends Expression implements LeafExpression {
-
- private final Map<String, String> keyValues;
-
- public Properties(Map<String, String> properties) {
- super(ImmutableList.of());
- this.keyValues = Objects.requireNonNull(properties, "properties can not be null");
- }
-
- public Map<String, String> getMap() {
- return keyValues;
- }
-
- @Override
- public boolean nullable() {
- return false;
- }
-
- @Override
- public DataType getDataType() throws UnboundException {
- return MapType.SYSTEM_DEFAULT;
- }
-
- @Override
- public String toSql() {
- return getMap()
- .entrySet()
- .stream()
- .map(kv -> "'" + kv.getKey() + "' = '" + kv.getValue() + "'")
- .collect(Collectors.joining(", "));
- }
-
- @Override
- public String toString() {
- return "Properties(" + toSql() + ")";
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- if (!super.equals(o)) {
- return false;
- }
- Properties that = (Properties) o;
- return Objects.equals(keyValues, that.keyValues);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(super.hashCode(), keyValues);
- }
-
- @Override
- public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
- return visitor.visitProperties(this, context);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Hdfs.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Hdfs.java
index c66684cd22c..d82878c4a48 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Hdfs.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Hdfs.java
@@ -19,7 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions.table;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.exceptions.AnalysisException;
-import org.apache.doris.nereids.trees.expressions.Properties;
+import org.apache.doris.nereids.trees.expressions.TVFProperties;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.coercion.AnyDataType;
import org.apache.doris.tablefunction.HdfsTableValuedFunction;
@@ -30,13 +30,13 @@ import java.util.Map;
/** hdfs */
public class Hdfs extends TableValuedFunction {
- public Hdfs(Properties properties) {
+ public Hdfs(TVFProperties properties) {
super("hdfs", properties);
}
@Override
public FunctionSignature customSignature() {
- return FunctionSignature.of(AnyDataType.INSTANCE_WITHOUT_INDEX, (List) getArgumentsTypes());
+ return FunctionSignature.of(AnyDataType.INSTANCE, (List) getArgumentsTypes());
}
@Override
@@ -46,7 +46,7 @@ public class Hdfs extends TableValuedFunction {
return new HdfsTableValuedFunction(arguments);
} catch (Throwable t) {
throw new AnalysisException("Can not build HdfsTableValuedFunction by "
- + this + ": " + t.getMessage(), t);
+ + this + ": " + t.getMessage(), t);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Local.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Local.java
index d45a4c93943..cd4169c1c95 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Local.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Local.java
@@ -19,7 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions.table;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.exceptions.AnalysisException;
-import org.apache.doris.nereids.trees.expressions.Properties;
+import org.apache.doris.nereids.trees.expressions.TVFProperties;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.coercion.AnyDataType;
import org.apache.doris.tablefunction.LocalTableValuedFunction;
@@ -31,13 +31,13 @@ import java.util.Map;
* local
*/
public class Local extends TableValuedFunction {
- public Local(Properties properties) {
+ public Local(TVFProperties properties) {
super("local", properties);
}
@Override
public FunctionSignature customSignature() {
- return FunctionSignature.of(AnyDataType.INSTANCE_WITHOUT_INDEX, getArgumentsTypes());
+ return FunctionSignature.of(AnyDataType.INSTANCE, getArgumentsTypes());
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Numbers.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Numbers.java
index c5febcf9749..0043ab9c1f7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Numbers.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Numbers.java
@@ -24,8 +24,8 @@ import org.apache.doris.common.NereidsException;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.Properties;
import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.TVFProperties;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.BigIntType;
import org.apache.doris.statistics.ColumnStatistic;
@@ -42,7 +42,7 @@ import java.util.Map;
/** Numbers */
public class Numbers extends TableValuedFunction {
- public Numbers(Properties properties) {
+ public Numbers(TVFProperties properties) {
super("numbers", properties);
}
@@ -72,7 +72,7 @@ public class Numbers extends TableValuedFunction {
Map<Expression, ColumnStatistic> columnToStatistics = Maps.newHashMap();
ColumnStatistic columnStat = new ColumnStatisticBuilder()
.setCount(rowNum).setNdv(rowNum).setAvgSizeByte(8).setNumNulls(0).setDataSize(8).setMinValue(0)
- .setMaxValue(rowNum - 1)
+ .setMaxValue(rowNum - 1).setSelectivity(1.0 / rowNum)
.setMinExpr(new IntLiteral(0, Type.BIGINT))
.setMaxExpr(new IntLiteral(rowNum - 1, Type.BIGINT))
.build();
@@ -101,7 +101,7 @@ public class Numbers extends TableValuedFunction {
@Override
public Numbers withChildren(List<Expression> children) {
Preconditions.checkArgument(children().size() == 1
- && children().get(0) instanceof Properties);
- return new Numbers((Properties) children.get(0));
+ && children().get(0) instanceof TVFProperties);
+ return new Numbers((TVFProperties) children.get(0));
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java
index 29d8ad08218..d871219b7a8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java
@@ -19,7 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions.table;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.exceptions.AnalysisException;
-import org.apache.doris.nereids.trees.expressions.Properties;
+import org.apache.doris.nereids.trees.expressions.TVFProperties;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.coercion.AnyDataType;
import org.apache.doris.tablefunction.S3TableValuedFunction;
@@ -29,13 +29,13 @@ import java.util.Map;
/** s3 */
public class S3 extends TableValuedFunction {
- public S3(Properties properties) {
+ public S3(TVFProperties properties) {
super("s3", properties);
}
@Override
public FunctionSignature customSignature() {
- return FunctionSignature.of(AnyDataType.INSTANCE_WITHOUT_INDEX, getArgumentsTypes());
+ return FunctionSignature.of(AnyDataType.INSTANCE, getArgumentsTypes());
}
@Override
@@ -45,7 +45,7 @@ public class S3 extends TableValuedFunction {
return new S3TableValuedFunction(arguments);
} catch (Throwable t) {
throw new AnalysisException("Can not build S3TableValuedFunction by "
- + this + ": " + t.getMessage(), t);
+ + this + ": " + t.getMessage(), t);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/TableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/TableValuedFunction.java
index 5acc73eb75a..6443eab9728 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/TableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/TableValuedFunction.java
@@ -23,8 +23,8 @@ import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.exceptions.UnboundException;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.Properties;
import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.TVFProperties;
import org.apache.doris.nereids.trees.expressions.functions.BoundFunction;
import org.apache.doris.nereids.trees.expressions.functions.CustomSignature;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
@@ -57,7 +57,7 @@ public abstract class TableValuedFunction extends BoundFunction implements Unary
}
});
- public TableValuedFunction(String functionName, Properties tvfProperties) {
+ public TableValuedFunction(String functionName, TVFProperties tvfProperties) {
super(functionName, tvfProperties);
}
@@ -78,8 +78,8 @@ public abstract class TableValuedFunction extends BoundFunction implements Unary
return new Statistics(0, columnToStatistics);
}
- public Properties getTVFProperties() {
- return (Properties) child(0);
+ public TVFProperties getTVFProperties() {
+ return (TVFProperties) child(0);
}
public final String getTableName() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java
index d95f35a6f69..4517f444e0b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java
@@ -63,7 +63,6 @@ import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.NullSafeEqual;
import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.OrderExpression;
-import org.apache.doris.nereids.trees.expressions.Properties;
import org.apache.doris.nereids.trees.expressions.ScalarSubquery;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
@@ -118,10 +117,6 @@ public abstract class ExpressionVisitor<R, C>
public abstract R visit(Expression expr, C context);
- public R visitProperties(Properties properties, C context) {
- return visit(properties, context);
- }
-
@Override
public R visitAggregateFunction(AggregateFunction aggregateFunction, C context) {
return visitBoundFunction(aggregateFunction, context);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/MapType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/MapType.java
index c576dcb9933..40e0dae0fdb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/MapType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/MapType.java
@@ -29,8 +29,6 @@ public class MapType extends DataType {
public static final MapType INSTANCE = new MapType();
- public static final MapType SYSTEM_DEFAULT = new MapType();
-
public static final int WIDTH = 24;
private MapType() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/AnyDataType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/AnyDataType.java
index e1097df65f1..be07a7ee7b8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/AnyDataType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/AnyDataType.java
@@ -26,18 +26,7 @@ import org.apache.doris.nereids.types.DataType;
*/
public class AnyDataType implements AbstractDataType {
- public static final AnyDataType INSTANCE_WITHOUT_INDEX = new AnyDataType(-1);
-
- public static final AnyDataType INSTANCE = new AnyDataType(-1);
-
- private final int index;
-
- public AnyDataType(int index) {
- if (index < 0) {
- index = -1;
- }
- this.index = index;
- }
+ public static final AnyDataType INSTANCE = new AnyDataType();
@Override
public DataType defaultConcreteType() {
@@ -58,9 +47,4 @@ public class AnyDataType implements AbstractDataType {
public String simpleString() {
return "any";
}
-
- public int getIndex() {
- return index;
- }
-
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
index 2703165b428..97dd6719e9b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
@@ -82,7 +82,6 @@ import org.apache.doris.policy.Policy;
import org.apache.doris.policy.StoragePolicy;
import org.apache.doris.resource.workloadgroup.WorkloadGroup;
import org.apache.doris.statistics.AnalysisInfo;
-import org.apache.doris.statistics.TableStatsMeta;
import org.apache.doris.system.Backend;
import org.apache.doris.system.Frontend;
import org.apache.doris.transaction.TransactionState;
@@ -1066,16 +1065,16 @@ public class EditLog {
env.getBinlogManager().addBarrierLog(log, logId);
break;
}
+ // For backward compatible with 2.0.3
case OperationType.OP_UPDATE_TABLE_STATS: {
- env.getAnalysisManager().replayUpdateTableStatsStatus((TableStatsMeta) journal.getData());
break;
}
+ // For backward compatible with 2.0.3
case OperationType.OP_PERSIST_AUTO_JOB: {
- env.getAnalysisManager().replayPersistSysJob((AnalysisInfo) journal.getData());
break;
}
+ // For backward compatible with 2.0.3
case OperationType.OP_DELETE_TABLE_STATS: {
- env.getAnalysisManager().replayTableStatsDeletion((TableStatsDeletionLog) journal.getData());
break;
}
default: {
@@ -1881,16 +1880,4 @@ public class EditLog {
LOG.info("logId {}, barrier {}", logId, log);
return logId;
}
-
- public void logCreateTableStats(TableStatsMeta tableStats) {
- logEdit(OperationType.OP_UPDATE_TABLE_STATS, tableStats);
- }
-
- public void logAutoJob(AnalysisInfo analysisInfo) {
- logEdit(OperationType.OP_PERSIST_AUTO_JOB, analysisInfo);
- }
-
- public void logDeleteTableStats(TableStatsDeletionLog log) {
- logEdit(OperationType.OP_DELETE_TABLE_STATS, log);
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/TableStatsDeletionLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/TableStatsDeletionLog.java
deleted file mode 100644
index 4016ff0139e..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/TableStatsDeletionLog.java
+++ /dev/null
@@ -1,47 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.persist;
-
-import org.apache.doris.common.io.Text;
-import org.apache.doris.common.io.Writable;
-import org.apache.doris.persist.gson.GsonUtils;
-
-import com.google.gson.annotations.SerializedName;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-public class TableStatsDeletionLog implements Writable {
-
- @SerializedName("id")
- public final long id;
-
- public TableStatsDeletionLog(long id) {
- this.id = id;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- Text.writeString(out, GsonUtils.GSON.toJson(this));
- }
-
- public static TableStatsDeletionLog read(DataInput dataInput) throws IOException {
- return GsonUtils.GSON.fromJson(Text.readString(dataInput), TableStatsDeletionLog.class);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java
deleted file mode 100644
index 40f870eee11..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java
+++ /dev/null
@@ -1,120 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.qe;
-
-import org.apache.doris.analysis.InsertStmt;
-import org.apache.doris.analysis.Queriable;
-import org.apache.doris.analysis.StatementBase;
-import org.apache.doris.catalog.Env;
-import org.apache.doris.cluster.ClusterNamespace;
-import org.apache.doris.common.Config;
-import org.apache.doris.common.util.DebugUtil;
-import org.apache.doris.metric.MetricRepo;
-import org.apache.doris.plugin.AuditEvent.EventType;
-import org.apache.doris.qe.QueryState.MysqlStateType;
-import org.apache.doris.service.FrontendOptions;
-
-import io.opentelemetry.api.trace.Span;
-import io.opentelemetry.api.trace.SpanContext;
-import io.opentelemetry.context.Context;
-import org.apache.commons.codec.digest.DigestUtils;
-
-public class AuditLogHelper {
-
- public static void logAuditLog(ConnectContext ctx, String origStmt, StatementBase parsedStmt,
- org.apache.doris.proto.Data.PQueryStatistics statistics, boolean printFuzzyVariables) {
- origStmt = origStmt.replace("\n", " ");
- // slow query
- long endTime = System.currentTimeMillis();
- long elapseMs = endTime - ctx.getStartTime();
- SpanContext spanContext = Span.fromContext(Context.current()).getSpanContext();
-
- ctx.getAuditEventBuilder().setEventType(EventType.AFTER_QUERY)
- .setDb(ClusterNamespace.getNameFromFullName(ctx.getDatabase()))
- .setState(ctx.getState().toString())
- .setErrorCode(ctx.getState().getErrorCode() == null ? 0 : ctx.getState().getErrorCode().getCode())
- .setErrorMessage((ctx.getState().getErrorMessage() == null ? "" :
- ctx.getState().getErrorMessage().replace("\n", " ").replace("\t", " ")))
- .setQueryTime(elapseMs)
- .setScanBytes(statistics == null ? 0 : statistics.getScanBytes())
- .setScanRows(statistics == null ? 0 : statistics.getScanRows())
- .setCpuTimeMs(statistics == null ? 0 : statistics.getCpuMs())
- .setPeakMemoryBytes(statistics == null ? 0 : statistics.getMaxPeakMemoryBytes())
- .setReturnRows(ctx.getReturnRows())
- .setStmtId(ctx.getStmtId())
- .setQueryId(ctx.queryId() == null ? "NaN" : DebugUtil.printId(ctx.queryId()))
- .setTraceId(spanContext.isValid() ? spanContext.getTraceId() : "")
- .setWorkloadGroup(ctx.getWorkloadGroupName())
- .setFuzzyVariables(!printFuzzyVariables ? "" : ctx.getSessionVariable().printFuzzyVariables());
-
- if (ctx.getState().isQuery()) {
- MetricRepo.COUNTER_QUERY_ALL.increase(1L);
- MetricRepo.USER_COUNTER_QUERY_ALL.getOrAdd(ctx.getQualifiedUser()).increase(1L);
- if (ctx.getState().getStateType() == MysqlStateType.ERR
- && ctx.getState().getErrType() != QueryState.ErrType.ANALYSIS_ERR) {
- // err query
- MetricRepo.COUNTER_QUERY_ERR.increase(1L);
- MetricRepo.USER_COUNTER_QUERY_ERR.getOrAdd(ctx.getQualifiedUser()).increase(1L);
- } else if (ctx.getState().getStateType() == MysqlStateType.OK
- || ctx.getState().getStateType() == MysqlStateType.EOF) {
- // ok query
- MetricRepo.HISTO_QUERY_LATENCY.update(elapseMs);
- MetricRepo.USER_HISTO_QUERY_LATENCY.getOrAdd(ctx.getQualifiedUser()).update(elapseMs);
-
- if (elapseMs > Config.qe_slow_log_ms) {
- String sqlDigest = DigestUtils.md5Hex(((Queriable) parsedStmt).toDigest());
- ctx.getAuditEventBuilder().setSqlDigest(sqlDigest);
- }
- }
- ctx.getAuditEventBuilder().setIsQuery(true);
- if (ctx.getQueryDetail() != null) {
- ctx.getQueryDetail().setEventTime(endTime);
- ctx.getQueryDetail().setEndTime(endTime);
- ctx.getQueryDetail().setLatency(elapseMs);
- ctx.getQueryDetail().setState(QueryDetail.QueryMemState.FINISHED);
- QueryDetailQueue.addOrUpdateQueryDetail(ctx.getQueryDetail());
- ctx.setQueryDetail(null);
- }
- } else {
- ctx.getAuditEventBuilder().setIsQuery(false);
- }
- ctx.getAuditEventBuilder().setIsNereids(ctx.getState().isNereids);
-
- ctx.getAuditEventBuilder().setFeIp(FrontendOptions.getLocalHostAddress());
-
- // We put origin query stmt at the end of audit log, for parsing the log more convenient.
- if (!ctx.getState().isQuery() && (parsedStmt != null && parsedStmt.needAuditEncryption())) {
- ctx.getAuditEventBuilder().setStmt(parsedStmt.toSql());
- } else {
- if (parsedStmt instanceof InsertStmt && !((InsertStmt) parsedStmt).needLoadManager()
- && ((InsertStmt) parsedStmt).isValuesOrConstantSelect()) {
- // INSERT INTO VALUES may be very long, so we only log at most 1K bytes.
- int length = Math.min(1024, origStmt.length());
- ctx.getAuditEventBuilder().setStmt(origStmt.substring(0, length));
- } else {
- ctx.getAuditEventBuilder().setStmt(origStmt);
- }
- }
- if (!Env.getCurrentEnv().isMaster()) {
- if (ctx.executor.isForwardToMaster()) {
- ctx.getAuditEventBuilder().setState(ctx.executor.getProxyStatus());
- }
- }
- Env.getCurrentAuditEventProcessor().handleAuditEvent(ctx.getAuditEventBuilder().build());
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java
index c06ddde23ac..29e29a9a7dd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java
@@ -39,6 +39,7 @@ import org.apache.doris.analysis.AlterResourceStmt;
import org.apache.doris.analysis.AlterRoutineLoadStmt;
import org.apache.doris.analysis.AlterSqlBlockRuleStmt;
import org.apache.doris.analysis.AlterSystemStmt;
+import org.apache.doris.analysis.AlterTableStatsStmt;
import org.apache.doris.analysis.AlterTableStmt;
import org.apache.doris.analysis.AlterUserStmt;
import org.apache.doris.analysis.AlterViewStmt;
@@ -160,6 +161,8 @@ public class DdlExecutor {
env.createMaterializedView((CreateMaterializedViewStmt) ddlStmt);
} else if (ddlStmt instanceof AlterTableStmt) {
env.alterTable((AlterTableStmt) ddlStmt);
+ } else if (ddlStmt instanceof AlterTableStatsStmt) {
+ StatisticsRepository.alterTableStatistics((AlterTableStatsStmt) ddlStmt);
} else if (ddlStmt instanceof AlterColumnStatsStmt) {
StatisticsRepository.alterColumnStatistics((AlterColumnStatsStmt) ddlStmt);
} else if (ddlStmt instanceof AlterViewStmt) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/InternalQueryExecutionException.java b/fe/fe-core/src/main/java/org/apache/doris/qe/InternalQueryExecutionException.java
deleted file mode 100644
index c368533c53d..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/InternalQueryExecutionException.java
+++ /dev/null
@@ -1,24 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.qe;
-
-public class InternalQueryExecutionException extends RuntimeException {
- public InternalQueryExecutionException(String message, Throwable cause) {
- super(message, cause);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 34518114453..d4247496548 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -403,8 +403,6 @@ public class SessionVariable implements Serializable, Writable {
public static final String TEST_QUERY_CACHE_HIT = "test_query_cache_hit";
- public static final String ENABLE_FULL_AUTO_ANALYZE = "enable_full_auto_analyze";
-
public static final List<String> DEBUG_VARIABLES = ImmutableList.of(
SKIP_DELETE_PREDICATE,
SKIP_DELETE_BITMAP,
@@ -1144,24 +1142,6 @@ public class SessionVariable implements Serializable, Writable {
options = {"none", "sql_cache", "partition_cache"})
public String testQueryCacheHit = "none";
- @VariableMgr.VarAttr(name = ENABLE_FULL_AUTO_ANALYZE,
- description = {"该参数控制是否开启自动收集", "Set false to disable auto analyze"},
- flag = VariableMgr.GLOBAL)
- public boolean enableFullAutoAnalyze = true;
-
- @VariableMgr.VarAttr(name = FULL_AUTO_ANALYZE_START_TIME, needForward = true, checker = "checkAnalyzeTimeFormat",
- description = {"该参数定义自动ANALYZE例程的开始时间",
- "This parameter defines the start time for the automatic ANALYZE routine."},
- flag = VariableMgr.GLOBAL)
- public String fullAutoAnalyzeStartTime = "00:00:00";
-
- @VariableMgr.VarAttr(name = FULL_AUTO_ANALYZE_END_TIME, needForward = true, checker = "checkAnalyzeTimeFormat",
- description = {"该参数定义自动ANALYZE例程的结束时间",
- "This parameter defines the end time for the automatic ANALYZE routine."},
- flag = VariableMgr.GLOBAL)
- public String fullAutoAnalyzeEndTime = "02:00:00";
-
-
// If this fe is in fuzzy mode, then will use initFuzzyModeVariables to generate some variables,
// not the default value set in the code.
public void initFuzzyModeVariables() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index f4f40047b75..06e8f4d3929 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -138,6 +138,7 @@ import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.TabletInvertedIndex;
import org.apache.doris.catalog.TabletMeta;
import org.apache.doris.catalog.View;
+import org.apache.doris.catalog.external.ExternalTable;
import org.apache.doris.catalog.external.HMSExternalTable;
import org.apache.doris.clone.DynamicPartitionScheduler;
import org.apache.doris.cluster.ClusterNamespace;
@@ -195,7 +196,7 @@ import org.apache.doris.statistics.AnalysisInfo;
import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.Histogram;
import org.apache.doris.statistics.StatisticsRepository;
-import org.apache.doris.statistics.TableStatsMeta;
+import org.apache.doris.statistics.TableStatistic;
import org.apache.doris.statistics.query.QueryStatsUtil;
import org.apache.doris.system.Backend;
import org.apache.doris.system.Diagnoser;
@@ -239,6 +240,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
@@ -2390,16 +2392,29 @@ public class ShowExecutor {
private void handleShowTableStats() {
ShowTableStatsStmt showTableStatsStmt = (ShowTableStatsStmt) stmt;
TableIf tableIf = showTableStatsStmt.getTable();
- TableStatsMeta tableStats = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(tableIf.getId());
- /*
- HMSExternalTable table will fetch row count from HMS
- or estimate with file size and schema if it's not analyzed.
- tableStats == null means it's not analyzed, in this case show the estimated row count.
- */
- if (tableStats == null && tableIf instanceof HMSExternalTable) {
- resultSet = showTableStatsStmt.constructResultSet(tableIf.estimatedRowCount());
- } else {
- resultSet = showTableStatsStmt.constructResultSet(tableStats);
+ long partitionId = showTableStatsStmt.getPartitionId();
+ boolean showCache = showTableStatsStmt.isCached();
+ try {
+ if (tableIf instanceof ExternalTable && showCache) {
+ Optional<TableStatistic> tableStatistics = Env.getCurrentEnv().getStatisticsCache().getTableStatistics(
+ tableIf.getDatabase().getCatalog().getId(),
+ tableIf.getDatabase().getId(),
+ tableIf.getId());
+ if (tableStatistics.isPresent()) {
+ resultSet = showTableStatsStmt.constructResultSet(tableStatistics.get());
+ } else {
+ resultSet = showTableStatsStmt.constructResultSet(TableStatistic.UNKNOWN);
+ }
+ } else if (partitionId > 0) {
+ TableStatistic partStats = StatisticsRepository.fetchTableLevelOfPartStats(partitionId);
+ resultSet = showTableStatsStmt.constructResultSet(partStats);
+ } else {
+ TableStatistic tableStats = StatisticsRepository.fetchTableLevelStats(tableIf.getId());
+ resultSet = showTableStatsStmt.constructResultSet(tableStats);
+ }
+ } catch (DdlException e) {
+ LOG.warn("Table statistics do not exist: {}", tableIf.getName());
+ resultSet = showTableStatsStmt.constructResultSet(TableStatistic.UNKNOWN);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index 5d90008ff75..197c5fc9088 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -142,8 +142,8 @@ import org.apache.doris.rewrite.ExprRewriter;
import org.apache.doris.rewrite.mvrewrite.MVSelectFailedException;
import org.apache.doris.rpc.RpcException;
import org.apache.doris.service.FrontendOptions;
-import org.apache.doris.statistics.ResultRow;
import org.apache.doris.statistics.util.InternalQueryBuffer;
+import org.apache.doris.statistics.util.InternalQueryResult.ResultRow;
import org.apache.doris.task.LoadEtlTask;
import org.apache.doris.thrift.TFileFormatType;
import org.apache.doris.thrift.TFileType;
@@ -2490,8 +2490,7 @@ public class StmtExecutor {
planner = new NereidsPlanner(statementContext);
planner.plan(parsedStmt, context.getSessionVariable().toThrift());
} catch (Exception e) {
- LOG.warn("Arrow Flight SQL fall back to legacy planner, because: {}",
- e.getMessage(), e);
+ LOG.warn("fall back to legacy planner, because: {}", e.getMessage(), e);
parsedStmt = null;
planner = null;
context.getState().setNereids(false);
@@ -2503,9 +2502,10 @@ public class StmtExecutor {
analyze(context.getSessionVariable().toThrift());
}
} catch (Exception e) {
- LOG.warn("Failed to run internal SQL: {}", originStmt, e);
- throw new RuntimeException("Failed to execute internal SQL. " + Util.getRootCauseMessage(e), e);
+ throw new RuntimeException("Failed to execute internal SQL. "
+ + Util.getRootCauseMessage(e) + " " + originStmt.toString(), e);
}
+ planner.getFragments();
RowBatch batch;
coord = new Coordinator(context, analyzer, planner, context.getStatsErrorEstimator());
profile.addExecutionProfile(coord.getExecutionProfile());
@@ -2513,7 +2513,8 @@ public class StmtExecutor {
QeProcessorImpl.INSTANCE.registerQuery(context.queryId(),
new QeProcessorImpl.QueryInfo(context, originStmt.originStmt, coord));
} catch (UserException e) {
- throw new RuntimeException("Failed to execute internal SQL. " + Util.getRootCauseMessage(e), e);
+ throw new RuntimeException("Failed to execute internal SQL. "
+ + " " + Util.getRootCauseMessage(e) + originStmt.toString(), e);
}
Span queryScheduleSpan = context.getTracer()
@@ -2522,7 +2523,8 @@ public class StmtExecutor {
coord.exec();
} catch (Exception e) {
queryScheduleSpan.recordException(e);
- throw new InternalQueryExecutionException(e.getMessage() + Util.getRootCauseMessage(e), e);
+ throw new RuntimeException("Failed to execute internal SQL. "
+ + Util.getRootCauseMessage(e) + " " + originStmt.toString(), e);
} finally {
queryScheduleSpan.end();
}
@@ -2539,19 +2541,21 @@ public class StmtExecutor {
}
} catch (Exception e) {
fetchResultSpan.recordException(e);
- throw new RuntimeException("Failed to fetch internal SQL result. " + Util.getRootCauseMessage(e), e);
+ throw new RuntimeException("Failed to execute internal SQL. " + Util.getRootCauseMessage(e) + " "
+ + originStmt.toString(), e);
} finally {
fetchResultSpan.end();
}
} finally {
- AuditLogHelper.logAuditLog(context, originStmt.toString(), parsedStmt, getQueryStatisticsForAuditLog(),
- true);
QeProcessorImpl.INSTANCE.unregisterQuery(context.queryId());
}
}
private List<ResultRow> convertResultBatchToResultRows(TResultBatch batch) {
List<String> columns = parsedStmt.getColLabels();
+ List<PrimitiveType> types = parsedStmt.getResultExprs().stream()
+ .map(e -> e.getType().getPrimitiveType())
+ .collect(Collectors.toList());
List<ResultRow> resultRows = new ArrayList<>();
List<ByteBuffer> rows = batch.getRows();
for (ByteBuffer buffer : rows) {
@@ -2562,7 +2566,8 @@ public class StmtExecutor {
String value = queryBuffer.readStringWithLength();
values.add(value);
}
- ResultRow resultRow = new ResultRow(values);
+
+ ResultRow resultRow = new ResultRow(columns, types, values);
resultRows.add(resultRow);
}
return resultRows;
@@ -2585,14 +2590,6 @@ public class StmtExecutor {
this.proxyResultSet = proxyResultSet;
}
- public ConnectContext getContext() {
- return context;
- }
-
- public OriginStatement getOriginStmt() {
- return originStmt;
- }
-
public String getOriginStmtInString() {
if (originStmt != null && originStmt.originStmt != null) {
return originStmt.originStmt;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
index e3d5c8a91b8..ee39582aac4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
@@ -22,21 +22,19 @@ import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.persist.gson.GsonUtils;
+import org.apache.doris.statistics.util.InternalQueryResult.ResultRow;
import org.apache.doris.statistics.util.StatisticsUtil;
import com.google.gson.Gson;
import com.google.gson.annotations.SerializedName;
import com.google.gson.reflect.TypeToken;
-import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.core.util.CronExpression;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.lang.reflect.Type;
-import java.text.ParseException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -48,7 +46,6 @@ public class AnalysisInfo implements Writable {
private static final Logger LOG = LogManager.getLogger(AnalysisInfo.class);
- // TODO: useless, remove it later
public enum AnalysisMode {
INCREMENTAL,
FULL
@@ -69,12 +66,10 @@ public class AnalysisInfo implements Writable {
// submit by user directly
MANUAL,
// submit by system automatically
- SYSTEM;
+ SYSTEM
}
public enum ScheduleType {
- // Job created by AutoCollector is also `ONCE` type, this is because it runs once only and should be removed
- // when its information is expired
ONCE,
PERIOD,
AUTOMATIC
@@ -100,7 +95,6 @@ public class AnalysisInfo implements Writable {
@SerializedName("tblName")
public final String tblName;
- // TODO: Map here is wired, List is enough
@SerializedName("colToPartitions")
public final Map<String, Set<String>> colToPartitions;
@@ -157,39 +151,24 @@ public class AnalysisInfo implements Writable {
// True means this task is a table level task for external table.
// This kind of task is mainly to collect the number of rows of a table.
@SerializedName("externalTableLevelTask")
- public final boolean externalTableLevelTask;
+ public boolean externalTableLevelTask;
@SerializedName("partitionOnly")
- public final boolean partitionOnly;
+ public boolean partitionOnly;
@SerializedName("samplingPartition")
- public final boolean samplingPartition;
-
- @SerializedName("isAllPartition")
- public final boolean isAllPartition;
-
- @SerializedName("partitionCount")
- public final long partitionCount;
+ public boolean samplingPartition;
// For serialize
@SerializedName("cronExpr")
public String cronExprStr;
- @SerializedName("progress")
- public String progress;
-
- public CronExpression cronExpression;
-
- @SerializedName("forceFull")
- public final boolean forceFull;
-
public AnalysisInfo(long jobId, long taskId, List<Long> taskIds, String catalogName, String dbName, String tblName,
Map<String, Set<String>> colToPartitions, Set<String> partitionNames, String colName, Long indexId,
JobType jobType, AnalysisMode analysisMode, AnalysisMethod analysisMethod, AnalysisType analysisType,
int samplePercent, int sampleRows, int maxBucketNum, long periodTimeInMs, String message,
long lastExecTimeInMs, long timeCostInMs, AnalysisState state, ScheduleType scheduleType,
- boolean isExternalTableLevelTask, boolean partitionOnly, boolean samplingPartition,
- boolean isAllPartition, long partitionCount, CronExpression cronExpression, boolean forceFull) {
+ boolean isExternalTableLevelTask, boolean partitionOnly, boolean samplingPartition) {
this.jobId = jobId;
this.taskId = taskId;
this.taskIds = taskIds;
@@ -216,13 +195,6 @@ public class AnalysisInfo implements Writable {
this.externalTableLevelTask = isExternalTableLevelTask;
this.partitionOnly = partitionOnly;
this.samplingPartition = samplingPartition;
- this.isAllPartition = isAllPartition;
- this.partitionCount = partitionCount;
- this.cronExpression = cronExpression;
- if (cronExpression != null) {
- this.cronExprStr = cronExpression.getCronExpression();
- }
- this.forceFull = forceFull;
}
@Override
@@ -233,11 +205,11 @@ public class AnalysisInfo implements Writable {
sj.add("DBName: " + dbName);
sj.add("TableName: " + tblName);
sj.add("ColumnName: " + colName);
- sj.add("TaskType: " + analysisType);
- sj.add("TaskMode: " + analysisMode);
- sj.add("TaskMethod: " + analysisMethod);
+ sj.add("TaskType: " + analysisType.toString());
+ sj.add("TaskMode: " + analysisMode.toString());
+ sj.add("TaskMethod: " + analysisMethod.toString());
sj.add("Message: " + message);
- sj.add("CurrentState: " + state);
+ sj.add("CurrentState: " + state.toString());
if (samplePercent > 0) {
sj.add("SamplePercent: " + samplePercent);
}
@@ -259,10 +231,6 @@ public class AnalysisInfo implements Writable {
if (periodTimeInMs > 0) {
sj.add("periodTimeInMs: " + StatisticsUtil.getReadableTime(periodTimeInMs));
}
- if (StringUtils.isNotEmpty(cronExprStr)) {
- sj.add("cronExpr: " + cronExprStr);
- }
- sj.add("forceFull: " + forceFull);
return sj.toString();
}
@@ -278,6 +246,60 @@ public class AnalysisInfo implements Writable {
taskIds.add(taskId);
}
+ // TODO: use thrift
+ public static AnalysisInfo fromResultRow(ResultRow resultRow) {
+ try {
+ AnalysisInfoBuilder analysisInfoBuilder = new AnalysisInfoBuilder();
+ long jobId = Long.parseLong(resultRow.getColumnValue("job_id"));
+ analysisInfoBuilder.setJobId(jobId);
+ long taskId = Long.parseLong(resultRow.getColumnValue("task_id"));
+ analysisInfoBuilder.setTaskId(taskId);
+ String catalogName = resultRow.getColumnValue("catalog_name");
+ analysisInfoBuilder.setCatalogName(catalogName);
+ String dbName = resultRow.getColumnValue("db_name");
+ analysisInfoBuilder.setDbName(dbName);
+ String tblName = resultRow.getColumnValue("tbl_name");
+ analysisInfoBuilder.setTblName(tblName);
+ String colName = resultRow.getColumnValue("col_name");
+ analysisInfoBuilder.setColName(colName);
+ long indexId = Long.parseLong(resultRow.getColumnValue("index_id"));
+ analysisInfoBuilder.setIndexId(indexId);
+ String partitionNames = resultRow.getColumnValue("col_partitions");
+ Map<String, Set<String>> colToPartitions = getColToPartition(partitionNames);
+ analysisInfoBuilder.setColToPartitions(colToPartitions);
+ String jobType = resultRow.getColumnValue("job_type");
+ analysisInfoBuilder.setJobType(JobType.valueOf(jobType));
+ String analysisType = resultRow.getColumnValue("analysis_type");
+ analysisInfoBuilder.setAnalysisType(AnalysisType.valueOf(analysisType));
+ String analysisMode = resultRow.getColumnValue("analysis_mode");
+ analysisInfoBuilder.setAnalysisMode(AnalysisMode.valueOf(analysisMode));
+ String analysisMethod = resultRow.getColumnValue("analysis_method");
+ analysisInfoBuilder.setAnalysisMethod(AnalysisMethod.valueOf(analysisMethod));
+ String scheduleType = resultRow.getColumnValue("schedule_type");
+ analysisInfoBuilder.setScheduleType(ScheduleType.valueOf(scheduleType));
+ String state = resultRow.getColumnValue("state");
+ analysisInfoBuilder.setState(AnalysisState.valueOf(state));
+ String samplePercent = resultRow.getColumnValue("sample_percent");
+ analysisInfoBuilder.setSamplePercent(StatisticsUtil.convertStrToInt(samplePercent));
+ String sampleRows = resultRow.getColumnValue("sample_rows");
+ analysisInfoBuilder.setSampleRows(StatisticsUtil.convertStrToInt(sampleRows));
+ String maxBucketNum = resultRow.getColumnValue("max_bucket_num");
+ analysisInfoBuilder.setMaxBucketNum(StatisticsUtil.convertStrToInt(maxBucketNum));
+ String periodTimeInMs = resultRow.getColumnValue("period_time_in_ms");
+ analysisInfoBuilder.setPeriodTimeInMs(StatisticsUtil.convertStrToInt(periodTimeInMs));
+ String lastExecTimeInMs = resultRow.getColumnValue("last_exec_time_in_ms");
+ analysisInfoBuilder.setLastExecTimeInMs(StatisticsUtil.convertStrToLong(lastExecTimeInMs));
+ String timeCostInMs = resultRow.getColumnValue("time_cost_in_ms");
+ analysisInfoBuilder.setTimeCostInMs(StatisticsUtil.convertStrToLong(timeCostInMs));
+ String message = resultRow.getColumnValue("message");
+ analysisInfoBuilder.setMessage(message);
+ return analysisInfoBuilder.build();
+ } catch (Exception e) {
+ LOG.warn("Failed to deserialize analysis task info.", e);
+ return null;
+ }
+ }
+
public String getColToPartitionStr() {
if (colToPartitions == null || colToPartitions.isEmpty()) {
return "";
@@ -340,15 +362,7 @@ public class AnalysisInfo implements Writable {
return analysisInfoBuilder.build();
} else {
String json = Text.readString(dataInput);
- AnalysisInfo analysisInfo = GsonUtils.GSON.fromJson(json, AnalysisInfo.class);
- if (analysisInfo.cronExprStr != null) {
- try {
- analysisInfo.cronExpression = new CronExpression(analysisInfo.cronExprStr);
- } catch (ParseException e) {
- LOG.warn("Cron expression of job is invalid, there is a bug", e);
- }
- }
- return analysisInfo;
+ return GsonUtils.GSON.fromJson(json, AnalysisInfo.class);
}
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
index 7e97b4d951f..2fd0e25d727 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
@@ -23,8 +23,6 @@ import org.apache.doris.statistics.AnalysisInfo.AnalysisType;
import org.apache.doris.statistics.AnalysisInfo.JobType;
import org.apache.doris.statistics.AnalysisInfo.ScheduleType;
-import org.apache.logging.log4j.core.util.CronExpression;
-
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -56,12 +54,6 @@ public class AnalysisInfoBuilder {
private boolean externalTableLevelTask;
private boolean partitionOnly;
private boolean samplingPartition;
- private boolean isAllPartition;
- private long partitionCount;
-
- private CronExpression cronExpression;
-
- private boolean forceFull;
public AnalysisInfoBuilder() {
}
@@ -93,10 +85,6 @@ public class AnalysisInfoBuilder {
externalTableLevelTask = info.externalTableLevelTask;
partitionOnly = info.partitionOnly;
samplingPartition = info.samplingPartition;
- isAllPartition = info.isAllPartition;
- partitionCount = info.partitionCount;
- cronExpression = info.cronExpression;
- forceFull = info.forceFull;
}
public AnalysisInfoBuilder setJobId(long jobId) {
@@ -229,30 +217,11 @@ public class AnalysisInfoBuilder {
return this;
}
- public AnalysisInfoBuilder setAllPartition(boolean isAllPartition) {
- this.isAllPartition = isAllPartition;
- return this;
- }
-
- public AnalysisInfoBuilder setPartitionCount(long partitionCount) {
- this.partitionCount = partitionCount;
- return this;
- }
-
- public void setCronExpression(CronExpression cronExpression) {
- this.cronExpression = cronExpression;
- }
-
- public void setForceFull(boolean forceFull) {
- this.forceFull = forceFull;
- }
-
public AnalysisInfo build() {
return new AnalysisInfo(jobId, taskId, taskIds, catalogName, dbName, tblName, colToPartitions, partitionNames,
colName, indexId, jobType, analysisMode, analysisMethod, analysisType, samplePercent,
sampleRows, maxBucketNum, periodTimeInMs, message, lastExecTimeInMs, timeCostInMs, state, scheduleType,
- externalTableLevelTask, partitionOnly, samplingPartition, isAllPartition, partitionCount,
- cronExpression, forceFull);
+ externalTableLevelTask, partitionOnly, samplingPartition);
}
public AnalysisInfoBuilder copy() {
@@ -279,10 +248,6 @@ public class AnalysisInfoBuilder {
.setTimeCostInMs(timeCostInMs)
.setState(state)
.setScheduleType(scheduleType)
- .setExternalTableLevelTask(externalTableLevelTask)
- .setSamplingPartition(samplingPartition)
- .setPartitionOnly(partitionOnly)
- .setAllPartition(isAllPartition)
- .setPartitionCount(partitionCount);
+ .setExternalTableLevelTask(externalTableLevelTask);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index 3eebb1ec9aa..5ac0c0cd524 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -18,7 +18,6 @@
package org.apache.doris.statistics;
import org.apache.doris.analysis.AnalyzeDBStmt;
-import org.apache.doris.analysis.AnalyzeProperties;
import org.apache.doris.analysis.AnalyzeStmt;
import org.apache.doris.analysis.AnalyzeTblStmt;
import org.apache.doris.analysis.DropAnalyzeJobStmt;
@@ -29,9 +28,12 @@ import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.MaterializedIndexMeta;
+import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.TableIf;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.View;
import org.apache.doris.catalog.external.ExternalTable;
import org.apache.doris.catalog.external.HMSExternalTable;
@@ -40,14 +42,12 @@ import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.ThreadPoolManager.BlockedPolicy;
-import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.Daemon;
import org.apache.doris.common.util.Util;
+import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.persist.AnalyzeDeletionLog;
-import org.apache.doris.persist.TableStatsDeletionLog;
-import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowResultSet;
import org.apache.doris.qe.ShowResultSetMetaData;
@@ -56,31 +56,25 @@ import org.apache.doris.statistics.AnalysisInfo.AnalysisMode;
import org.apache.doris.statistics.AnalysisInfo.AnalysisType;
import org.apache.doris.statistics.AnalysisInfo.JobType;
import org.apache.doris.statistics.AnalysisInfo.ScheduleType;
-import org.apache.doris.statistics.util.SimpleQueue;
import org.apache.doris.statistics.util.StatisticsUtil;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
-import com.google.common.reflect.TypeToken;
+import com.google.common.collect.Maps;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.core.util.CronExpression;
import org.jetbrains.annotations.Nullable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
-import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -99,141 +93,26 @@ import java.util.stream.Collectors;
public class AnalysisManager extends Daemon implements Writable {
+ public AnalysisTaskScheduler taskScheduler;
+
private static final Logger LOG = LogManager.getLogger(AnalysisManager.class);
- // Tracking running manually submitted async tasks, keep in mem only
- protected final ConcurrentMap<Long, Map<Long, BaseAnalysisTask>> analysisJobIdToTaskMap = new ConcurrentHashMap<>();
+ private ConcurrentMap<Long, Map<Long, BaseAnalysisTask>> analysisJobIdToTaskMap = new ConcurrentHashMap<>();
private StatisticsCache statisticsCache;
private AnalysisTaskExecutor taskExecutor;
- // Store task information in metadata.
private final Map<Long, AnalysisInfo> analysisTaskInfoMap = Collections.synchronizedMap(new TreeMap<>());
-
- // Store job information in metadata
private final Map<Long, AnalysisInfo> analysisJobInfoMap = Collections.synchronizedMap(new TreeMap<>());
- // Tracking system submitted job, keep in mem only
- protected final Map<Long, AnalysisInfo> systemJobInfoMap = new ConcurrentHashMap<>();
-
- // Tracking and control sync analyze tasks, keep in mem only
private final ConcurrentMap<ConnectContext, SyncTaskCollection> ctxToSyncTask = new ConcurrentHashMap<>();
- private final Map<Long, TableStatsMeta> idToTblStats = new ConcurrentHashMap<>();
-
- protected SimpleQueue<AnalysisInfo> autoJobs = createSimpleQueue(null, this);
-
- private final Function<TaskStatusWrapper, Void> userJobStatusUpdater = w -> {
- AnalysisInfo info = w.info;
- AnalysisState taskState = w.taskState;
- String message = w.message;
- long time = w.time;
- if (analysisJobIdToTaskMap.get(info.jobId) == null) {
- return null;
- }
- info.state = taskState;
- info.message = message;
- // Update the task cost time when task finished or failed. And only log the final state.
- if (taskState.equals(AnalysisState.FINISHED) || taskState.equals(AnalysisState.FAILED)) {
- info.timeCostInMs = time - info.lastExecTimeInMs;
- info.lastExecTimeInMs = time;
- logCreateAnalysisTask(info);
- }
- info.lastExecTimeInMs = time;
- AnalysisInfo job = analysisJobInfoMap.get(info.jobId);
- // Job may get deleted during execution.
- if (job == null) {
- return null;
- }
- // Synchronize the job state change in job level.
- synchronized (job) {
- job.lastExecTimeInMs = time;
- // Set the job state to RUNNING when its first task becomes RUNNING.
- if (info.state.equals(AnalysisState.RUNNING) && job.state.equals(AnalysisState.PENDING)) {
- job.state = AnalysisState.RUNNING;
- replayCreateAnalysisJob(job);
- }
- boolean allFinished = true;
- boolean hasFailure = false;
- for (BaseAnalysisTask task : analysisJobIdToTaskMap.get(info.jobId).values()) {
- AnalysisInfo taskInfo = task.info;
- if (taskInfo.state.equals(AnalysisState.RUNNING) || taskInfo.state.equals(AnalysisState.PENDING)) {
- allFinished = false;
- break;
- }
- if (taskInfo.state.equals(AnalysisState.FAILED)) {
- hasFailure = true;
- }
- }
- if (allFinished) {
- if (hasFailure) {
- job.state = AnalysisState.FAILED;
- } else {
- job.state = AnalysisState.FINISHED;
- try {
- updateTableStats(job);
- } catch (Throwable e) {
- LOG.warn("Failed to update Table statistics in job: {}", info.toString(), e);
- }
- }
- logCreateAnalysisJob(job);
- analysisJobIdToTaskMap.remove(job.jobId);
- }
- }
- return null;
- };
-
- private final String progressDisplayTemplate = "%d Finished | %d Failed | %d In Progress | %d Total";
-
- protected final Function<TaskStatusWrapper, Void> systemJobStatusUpdater = w -> {
- AnalysisInfo info = w.info;
- info.state = w.taskState;
- info.message = w.message;
- AnalysisInfo job = systemJobInfoMap.get(info.jobId);
- if (job == null) {
- return null;
- }
- int failedCount = 0;
- StringJoiner reason = new StringJoiner(", ");
- Map<Long, BaseAnalysisTask> taskMap = analysisJobIdToTaskMap.get(info.jobId);
- for (BaseAnalysisTask task : taskMap.values()) {
- if (task.info.state.equals(AnalysisState.RUNNING) || task.info.state.equals(AnalysisState.PENDING)) {
- return null;
- }
- if (task.info.state.equals(AnalysisState.FAILED)) {
- failedCount++;
- reason.add(task.info.message);
- }
- }
- try {
- updateTableStats(job);
- } catch (Throwable e) {
- LOG.warn("Failed to update Table statistics in job: {}", info.toString(), e);
- } finally {
- job.lastExecTimeInMs = System.currentTimeMillis();
- job.message = reason.toString();
- job.progress = String.format(progressDisplayTemplate,
- taskMap.size() - failedCount, failedCount, 0, taskMap.size());
- if (failedCount > 0) {
- job.message = reason.toString();
- job.state = AnalysisState.FAILED;
- } else {
- job.state = AnalysisState.FINISHED;
- }
- autoJobs.offer(job);
- systemJobInfoMap.remove(info.jobId);
- }
- return null;
- };
-
- private final Function<TaskStatusWrapper, Void>[] updaters =
- new Function[] {userJobStatusUpdater, systemJobStatusUpdater};
-
public AnalysisManager() {
super(TimeUnit.SECONDS.toMillis(StatisticConstants.ANALYZE_MANAGER_INTERVAL_IN_SECS));
if (!Env.isCheckpointThread()) {
- this.taskExecutor = new AnalysisTaskExecutor(Config.statistics_simultaneously_running_task_num);
+ this.taskScheduler = new AnalysisTaskScheduler();
+ this.taskExecutor = new AnalysisTaskExecutor(taskScheduler);
this.statisticsCache = new StatisticsCache();
taskExecutor.start();
}
@@ -245,7 +124,7 @@ public class AnalysisManager extends Daemon implements Writable {
}
private void clear() {
- clearExpiredAnalysisInfo(analysisJobInfoMap, (a) ->
+ clearMeta(analysisJobInfoMap, (a) ->
a.scheduleType.equals(ScheduleType.ONCE)
&& System.currentTimeMillis() - a.lastExecTimeInMs
> TimeUnit.DAYS.toMillis(StatisticConstants.ANALYSIS_JOB_INFO_EXPIRATION_TIME_IN_DAYS),
@@ -253,7 +132,7 @@ public class AnalysisManager extends Daemon implements Writable {
Env.getCurrentEnv().getEditLog().logDeleteAnalysisJob(new AnalyzeDeletionLog(id));
return null;
});
- clearExpiredAnalysisInfo(analysisTaskInfoMap, (a) -> System.currentTimeMillis() - a.lastExecTimeInMs
+ clearMeta(analysisTaskInfoMap, (a) -> System.currentTimeMillis() - a.lastExecTimeInMs
> TimeUnit.DAYS.toMillis(StatisticConstants.ANALYSIS_JOB_INFO_EXPIRATION_TIME_IN_DAYS),
(id) -> {
Env.getCurrentEnv().getEditLog().logDeleteAnalysisTask(new AnalyzeDeletionLog(id));
@@ -261,7 +140,7 @@ public class AnalysisManager extends Daemon implements Writable {
});
}
- private void clearExpiredAnalysisInfo(Map<Long, AnalysisInfo> infoMap, Predicate<AnalysisInfo> isExpired,
+ private void clearMeta(Map<Long, AnalysisInfo> infoMap, Predicate<AnalysisInfo> isExpired,
Function<Long, Void> writeLog) {
synchronized (infoMap) {
List<Long> expired = new ArrayList<>();
@@ -282,9 +161,6 @@ public class AnalysisManager extends Daemon implements Writable {
}
public void createAnalyze(AnalyzeStmt analyzeStmt, boolean proxy) throws DdlException {
- if (!StatisticsUtil.statsTblAvailable() && !FeConstants.runningUnitTest) {
- throw new DdlException("Stats table not available, please make sure your cluster status is normal");
- }
if (analyzeStmt instanceof AnalyzeDBStmt) {
createAnalysisJobs((AnalyzeDBStmt) analyzeStmt, proxy);
} else if (analyzeStmt instanceof AnalyzeTblStmt) {
@@ -294,53 +170,38 @@ public class AnalysisManager extends Daemon implements Writable {
public void createAnalysisJobs(AnalyzeDBStmt analyzeDBStmt, boolean proxy) throws DdlException {
DatabaseIf<TableIf> db = analyzeDBStmt.getDb();
- // Using auto analyzer if user specifies.
- if (analyzeDBStmt.getAnalyzeProperties().getProperties().containsKey("use.auto.analyzer")) {
- Env.getCurrentEnv().getStatisticsAutoCollector().analyzeDb(db);
- return;
- }
- List<AnalysisInfo> analysisInfos = buildAnalysisInfosForDB(db, analyzeDBStmt.getAnalyzeProperties());
- if (!analyzeDBStmt.isSync()) {
- sendJobId(analysisInfos, proxy);
- }
- }
-
- public List<AnalysisInfo> buildAnalysisInfosForDB(DatabaseIf<TableIf> db, AnalyzeProperties analyzeProperties) {
- db.readLock();
List<TableIf> tbls = db.getTables();
List<AnalysisInfo> analysisInfos = new ArrayList<>();
+ db.readLock();
try {
List<AnalyzeTblStmt> analyzeStmts = new ArrayList<>();
for (TableIf table : tbls) {
if (table instanceof View) {
continue;
}
- TableName tableName = new TableName(db.getCatalog().getName(), db.getFullName(),
+ TableName tableName = new TableName(analyzeDBStmt.getCtlIf().getName(), db.getFullName(),
table.getName());
// columnNames null means to add all visitable columns.
// Will get all the visible columns in analyzeTblStmt.check()
- AnalyzeTblStmt analyzeTblStmt = new AnalyzeTblStmt(analyzeProperties, tableName,
+ AnalyzeTblStmt analyzeTblStmt = new AnalyzeTblStmt(analyzeDBStmt.getAnalyzeProperties(), tableName,
null, db.getId(), table);
try {
analyzeTblStmt.check();
} catch (AnalysisException analysisException) {
- LOG.warn("Failed to build analyze job: {}",
- analysisException.getMessage(), analysisException);
+ throw new DdlException(analysisException.getMessage(), analysisException);
}
analyzeStmts.add(analyzeTblStmt);
}
for (AnalyzeTblStmt analyzeTblStmt : analyzeStmts) {
- try {
- analysisInfos.add(buildAndAssignJob(analyzeTblStmt));
- } catch (DdlException e) {
- LOG.warn("Failed to build analyze job: {}",
- e.getMessage(), e);
- }
+ analysisInfos.add(buildAndAssignJob(analyzeTblStmt));
+ }
+ if (!analyzeDBStmt.isSync()) {
+ sendJobId(analysisInfos, proxy);
}
} finally {
db.readUnlock();
}
- return analysisInfos;
+
}
// Each analyze stmt corresponding to an analysis job.
@@ -353,8 +214,11 @@ public class AnalysisManager extends Daemon implements Writable {
}
@Nullable
- @VisibleForTesting
- protected AnalysisInfo buildAndAssignJob(AnalyzeTblStmt stmt) throws DdlException {
+ private AnalysisInfo buildAndAssignJob(AnalyzeTblStmt stmt) throws DdlException {
+ if (!StatisticsUtil.statsTblAvailable() && !FeConstants.runningUnitTest) {
+ throw new DdlException("Stats table not available, please make sure your cluster status is normal");
+ }
+
AnalysisInfo jobInfo = buildAnalysisJobInfo(stmt);
if (jobInfo.colToPartitions.isEmpty()) {
// No statistics need to be collected or updated
@@ -364,44 +228,66 @@ public class AnalysisManager extends Daemon implements Writable {
boolean isSync = stmt.isSync();
Map<Long, BaseAnalysisTask> analysisTaskInfos = new HashMap<>();
createTaskForEachColumns(jobInfo, analysisTaskInfos, isSync);
- if (!jobInfo.partitionOnly && stmt.isAllColumns()
- && StatisticsUtil.isExternalTable(jobInfo.catalogName, jobInfo.dbName, jobInfo.tblName)) {
- createTableLevelTaskForExternalTable(jobInfo, analysisTaskInfos, isSync);
+ createTaskForMVIdx(jobInfo, analysisTaskInfos, isSync);
+ if (stmt.isAllColumns()) {
+ createTaskForExternalTable(jobInfo, analysisTaskInfos, isSync);
+ }
+ if (!isSync) {
+ persistAnalysisJob(jobInfo);
+ analysisJobIdToTaskMap.put(jobInfo.jobId, analysisTaskInfos);
}
+ if (!isSync) {
+ try {
+ updateTableStats(jobInfo);
+ } catch (Throwable e) {
+ throw new DdlException("Failed to update Table statistics");
+ }
+ }
+
if (isSync) {
syncExecute(analysisTaskInfos.values());
- updateTableStats(jobInfo);
return null;
}
- recordAnalysisJob(jobInfo);
- analysisJobIdToTaskMap.put(jobInfo.jobId, analysisTaskInfos);
- // TODO: maybe we should update table stats only when all task succeeded.
- updateTableStats(jobInfo);
- if (!jobInfo.scheduleType.equals(ScheduleType.PERIOD)) {
- analysisTaskInfos.values().forEach(taskExecutor::submitTask);
- }
+
+ analysisTaskInfos.values().forEach(taskScheduler::schedule);
return jobInfo;
}
+ // Analysis job created by the system
+ public void createAnalysisJob(AnalysisInfo info) throws DdlException {
+ AnalysisInfo jobInfo = buildAnalysisJobInfo(info);
+ if (jobInfo.colToPartitions.isEmpty()) {
+ // No statistics need to be collected or updated
+ return;
+ }
+
+ Map<Long, BaseAnalysisTask> analysisTaskInfos = new HashMap<>();
+ createTaskForEachColumns(jobInfo, analysisTaskInfos, false);
+ createTaskForMVIdx(jobInfo, analysisTaskInfos, false);
+ if (!jobInfo.jobType.equals(JobType.SYSTEM)) {
+ persistAnalysisJob(jobInfo);
+ analysisJobIdToTaskMap.put(jobInfo.jobId, analysisTaskInfos);
+ }
+
+ analysisTaskInfos.values().forEach(taskScheduler::schedule);
+ }
+
private void sendJobId(List<AnalysisInfo> analysisInfos, boolean proxy) {
List<Column> columns = new ArrayList<>();
- columns.add(new Column("Job_Id", ScalarType.createVarchar(19)));
columns.add(new Column("Catalog_Name", ScalarType.createVarchar(1024)));
columns.add(new Column("DB_Name", ScalarType.createVarchar(1024)));
columns.add(new Column("Table_Name", ScalarType.createVarchar(1024)));
columns.add(new Column("Columns", ScalarType.createVarchar(1024)));
+ columns.add(new Column("Job_Id", ScalarType.createVarchar(19)));
ShowResultSetMetaData commonResultSetMetaData = new ShowResultSetMetaData(columns);
List<List<String>> resultRows = new ArrayList<>();
for (AnalysisInfo analysisInfo : analysisInfos) {
- if (analysisInfo == null) {
- continue;
- }
List<String> row = new ArrayList<>();
- row.add(String.valueOf(analysisInfo.jobId));
row.add(analysisInfo.catalogName);
row.add(analysisInfo.dbName);
row.add(analysisInfo.tblName);
row.add(analysisInfo.colName);
+ row.add(String.valueOf(analysisInfo.jobId));
resultRows.add(row);
}
ShowResultSet commonResultSet = new ShowResultSet(commonResultSetMetaData, resultRows);
@@ -431,13 +317,13 @@ public class AnalysisManager extends Daemon implements Writable {
* TODO Supports incremental collection of statistics from materialized views
*/
private Map<String, Set<String>> validateAndGetPartitions(TableIf table, Set<String> columnNames,
- Set<String> partitionNames, AnalysisType analysisType) throws DdlException {
+ Set<String> partitionNames, AnalysisType analysisType, AnalysisMode analysisMode) throws DdlException {
long tableId = table.getId();
Map<String, Set<String>> columnToPartitions = columnNames.stream()
.collect(Collectors.toMap(
columnName -> columnName,
- columnName -> new HashSet<>(partitionNames == null ? Collections.emptySet() : partitionNames)
+ columnName -> new HashSet<>(partitionNames)
));
if (analysisType == AnalysisType.HISTOGRAM) {
@@ -454,7 +340,7 @@ public class AnalysisManager extends Daemon implements Writable {
}
// Get the partition granularity statistics that have been collected
- Map<String, Set<String>> existColAndPartsForStats = StatisticsRepository
+ Map<String, Set<Long>> existColAndPartsForStats = StatisticsRepository
.fetchColAndPartsForStats(tableId);
if (existColAndPartsForStats.isEmpty()) {
@@ -462,30 +348,42 @@ public class AnalysisManager extends Daemon implements Writable {
return columnToPartitions;
}
- Set<String> existPartIdsForStats = new HashSet<>();
+ Set<Long> existPartIdsForStats = new HashSet<>();
existColAndPartsForStats.values().forEach(existPartIdsForStats::addAll);
- Set<String> idToPartition = StatisticsUtil.getPartitionIds(table);
+ Map<Long, String> idToPartition = StatisticsUtil.getPartitionIdToName(table);
// Get an invalid set of partitions (those partitions were deleted)
- Set<String> invalidPartIds = existPartIdsForStats.stream()
- .filter(id -> !idToPartition.contains(id)).collect(Collectors.toSet());
+ Set<Long> invalidPartIds = existPartIdsForStats.stream()
+ .filter(id -> !idToPartition.containsKey(id)).collect(Collectors.toSet());
if (!invalidPartIds.isEmpty()) {
// Delete invalid partition statistics to avoid affecting table statistics
StatisticsRepository.dropStatistics(invalidPartIds);
}
- if (analysisType == AnalysisType.FUNDAMENTALS) {
- return table.findReAnalyzeNeededPartitions();
+ if (analysisMode == AnalysisMode.INCREMENTAL && analysisType == AnalysisType.FUNDAMENTALS) {
+ existColAndPartsForStats.values().forEach(partIds -> partIds.removeAll(invalidPartIds));
+ // In incremental collection mode, just collect the uncollected partition statistics
+ existColAndPartsForStats.forEach((columnName, partitionIds) -> {
+ Set<String> existPartitions = partitionIds.stream()
+ .map(idToPartition::get)
+ .collect(Collectors.toSet());
+ columnToPartitions.computeIfPresent(columnName, (colName, partNames) -> {
+ partNames.removeAll(existPartitions);
+ return partNames;
+ });
+ });
+ if (invalidPartIds.isEmpty()) {
+ // There is no invalid statistics, so there is no need to update table statistics,
+ // remove columns that do not require re-collection of statistics
+ columnToPartitions.entrySet().removeIf(entry -> entry.getValue().isEmpty());
+ }
}
return columnToPartitions;
}
- // Make sure colName of job has all the column as this AnalyzeStmt specified, no matter whether it will be analyzed
- // or not.
- @VisibleForTesting
- public AnalysisInfo buildAnalysisJobInfo(AnalyzeTblStmt stmt) throws DdlException {
- AnalysisInfoBuilder infoBuilder = new AnalysisInfoBuilder();
+ private AnalysisInfo buildAnalysisJobInfo(AnalyzeTblStmt stmt) throws DdlException {
+ AnalysisInfoBuilder taskInfoBuilder = new AnalysisInfoBuilder();
long jobId = Env.getCurrentEnv().getNextId();
String catalogName = stmt.getCatalogName();
String db = stmt.getDBName();
@@ -497,75 +395,140 @@ public class AnalysisManager extends Daemon implements Writable {
Set<String> partitionNames = stmt.getPartitionNames();
boolean partitionOnly = stmt.isPartitionOnly();
boolean isSamplingPartition = stmt.isSamplingPartition();
- boolean isAllPartition = stmt.isAllPartitions();
- long partitionCount = stmt.getPartitionCount();
int samplePercent = stmt.getSamplePercent();
int sampleRows = stmt.getSampleRows();
AnalysisType analysisType = stmt.getAnalysisType();
AnalysisMode analysisMode = stmt.getAnalysisMode();
AnalysisMethod analysisMethod = stmt.getAnalysisMethod();
ScheduleType scheduleType = stmt.getScheduleType();
- CronExpression cronExpression = stmt.getCron();
- infoBuilder.setJobId(jobId);
- infoBuilder.setCatalogName(catalogName);
- infoBuilder.setDbName(db);
- infoBuilder.setTblName(tblName);
- // TODO: Refactor later, DON'T MODIFY IT RIGHT NOW
+ taskInfoBuilder.setJobId(jobId);
+ taskInfoBuilder.setCatalogName(catalogName);
+ taskInfoBuilder.setDbName(db);
+ taskInfoBuilder.setTblName(tblName);
StringJoiner stringJoiner = new StringJoiner(",", "[", "]");
for (String colName : columnNames) {
stringJoiner.add(colName);
}
- infoBuilder.setColName(stringJoiner.toString());
- infoBuilder.setPartitionNames(partitionNames);
- infoBuilder.setPartitionOnly(partitionOnly);
- infoBuilder.setSamplingPartition(isSamplingPartition);
- infoBuilder.setAllPartition(isAllPartition);
- infoBuilder.setPartitionCount(partitionCount);
- infoBuilder.setJobType(JobType.MANUAL);
- infoBuilder.setState(AnalysisState.PENDING);
- infoBuilder.setLastExecTimeInMs(System.currentTimeMillis());
- infoBuilder.setAnalysisType(analysisType);
- infoBuilder.setAnalysisMode(analysisMode);
- infoBuilder.setAnalysisMethod(analysisMethod);
- infoBuilder.setScheduleType(scheduleType);
- infoBuilder.setLastExecTimeInMs(0);
- infoBuilder.setCronExpression(cronExpression);
- infoBuilder.setForceFull(stmt.forceFull());
+ taskInfoBuilder.setColName(stringJoiner.toString());
+ taskInfoBuilder.setPartitionNames(partitionNames);
+ taskInfoBuilder.setPartitionOnly(partitionOnly);
+ taskInfoBuilder.setSamplingPartition(isSamplingPartition);
+ taskInfoBuilder.setJobType(JobType.MANUAL);
+ taskInfoBuilder.setState(AnalysisState.PENDING);
+ taskInfoBuilder.setLastExecTimeInMs(System.currentTimeMillis());
+ taskInfoBuilder.setAnalysisType(analysisType);
+ taskInfoBuilder.setAnalysisMode(analysisMode);
+ taskInfoBuilder.setAnalysisMethod(analysisMethod);
+ taskInfoBuilder.setScheduleType(scheduleType);
+ taskInfoBuilder.setLastExecTimeInMs(0);
+
if (analysisMethod == AnalysisMethod.SAMPLE) {
- infoBuilder.setSamplePercent(samplePercent);
- infoBuilder.setSampleRows(sampleRows);
+ taskInfoBuilder.setSamplePercent(samplePercent);
+ taskInfoBuilder.setSampleRows(sampleRows);
}
if (analysisType == AnalysisType.HISTOGRAM) {
int numBuckets = stmt.getNumBuckets();
int maxBucketNum = numBuckets > 0 ? numBuckets
: StatisticConstants.HISTOGRAM_MAX_BUCKET_NUM;
- infoBuilder.setMaxBucketNum(maxBucketNum);
+ taskInfoBuilder.setMaxBucketNum(maxBucketNum);
}
- long periodTimeInMs = stmt.getPeriodTimeInMs();
- infoBuilder.setPeriodTimeInMs(periodTimeInMs);
+ if (scheduleType == ScheduleType.PERIOD) {
+ long periodTimeInMs = stmt.getPeriodTimeInMs();
+ taskInfoBuilder.setPeriodTimeInMs(periodTimeInMs);
+ }
Map<String, Set<String>> colToPartitions = validateAndGetPartitions(table, columnNames,
- partitionNames, analysisType);
- infoBuilder.setColToPartitions(colToPartitions);
- infoBuilder.setTaskIds(Lists.newArrayList());
-
- return infoBuilder.build();
+ partitionNames, analysisType, analysisMode);
+ taskInfoBuilder.setColToPartitions(colToPartitions);
+ taskInfoBuilder.setTaskIds(Lists.newArrayList());
+
+ return taskInfoBuilder.build();
+ }
+
+ private AnalysisInfo buildAnalysisJobInfo(AnalysisInfo jobInfo) {
+ AnalysisInfoBuilder taskInfoBuilder = new AnalysisInfoBuilder();
+ taskInfoBuilder.setJobId(jobInfo.jobId);
+ taskInfoBuilder.setCatalogName(jobInfo.catalogName);
+ taskInfoBuilder.setDbName(jobInfo.dbName);
+ taskInfoBuilder.setTblName(jobInfo.tblName);
+ taskInfoBuilder.setJobType(JobType.SYSTEM);
+ taskInfoBuilder.setState(AnalysisState.PENDING);
+ taskInfoBuilder.setLastExecTimeInMs(System.currentTimeMillis());
+ taskInfoBuilder.setAnalysisType(jobInfo.analysisType);
+ taskInfoBuilder.setAnalysisMode(jobInfo.analysisMode);
+ taskInfoBuilder.setAnalysisMethod(jobInfo.analysisMethod);
+ taskInfoBuilder.setScheduleType(jobInfo.scheduleType);
+ taskInfoBuilder.setSamplePercent(jobInfo.samplePercent);
+ taskInfoBuilder.setSampleRows(jobInfo.sampleRows);
+ taskInfoBuilder.setMaxBucketNum(jobInfo.maxBucketNum);
+ taskInfoBuilder.setPeriodTimeInMs(jobInfo.periodTimeInMs);
+ taskInfoBuilder.setLastExecTimeInMs(jobInfo.lastExecTimeInMs);
+ try {
+ TableIf table = StatisticsUtil
+ .findTable(jobInfo.catalogName, jobInfo.dbName, jobInfo.tblName);
+ Map<String, Set<String>> colToPartitions = validateAndGetPartitions(table, jobInfo.colToPartitions.keySet(),
+ jobInfo.partitionNames, jobInfo.analysisType, jobInfo.analysisMode);
+ taskInfoBuilder.setColToPartitions(colToPartitions);
+ } catch (Throwable e) {
+ throw new RuntimeException(e);
+ }
+ return taskInfoBuilder.build();
}
- @VisibleForTesting
- public void recordAnalysisJob(AnalysisInfo jobInfo) throws DdlException {
+ private void persistAnalysisJob(AnalysisInfo jobInfo) throws DdlException {
if (jobInfo.scheduleType == ScheduleType.PERIOD && jobInfo.lastExecTimeInMs > 0) {
return;
}
AnalysisInfoBuilder jobInfoBuilder = new AnalysisInfoBuilder(jobInfo);
AnalysisInfo analysisInfo = jobInfoBuilder.setTaskId(-1).build();
- replayCreateAnalysisJob(analysisInfo);
+ logCreateAnalysisJob(analysisInfo);
+ }
+
+ private void createTaskForMVIdx(AnalysisInfo jobInfo, Map<Long, BaseAnalysisTask> analysisTasks,
+ boolean isSync) throws DdlException {
+ TableIf table;
+ try {
+ table = StatisticsUtil.findTable(jobInfo.catalogName, jobInfo.dbName, jobInfo.tblName);
+ } catch (Throwable e) {
+ LOG.warn(e.getMessage());
+ return;
+ }
+
+ TableType type = table.getType();
+ if (jobInfo.analysisType != AnalysisType.INDEX || !type.equals(TableType.OLAP)) {
+ // not need to collect statistics for materialized view
+ return;
+ }
+
+ OlapTable olapTable = (OlapTable) table;
+
+ try {
+ olapTable.readLock();
+ for (MaterializedIndexMeta meta : olapTable.getIndexIdToMeta().values()) {
+ if (meta.getDefineStmt() == null) {
+ continue;
+ }
+ long indexId = meta.getIndexId();
+ long taskId = Env.getCurrentEnv().getNextId();
+ AnalysisInfoBuilder indexTaskInfoBuilder = new AnalysisInfoBuilder(jobInfo);
+ AnalysisInfo analysisInfo = indexTaskInfoBuilder.setIndexId(indexId)
+ .setTaskId(taskId).setLastExecTimeInMs(System.currentTimeMillis()).build();
+ jobInfo.addTaskId(taskId);
+ if (isSync) {
+ return;
+ }
+ analysisTasks.put(taskId, createTask(analysisInfo));
+ logCreateAnalysisTask(analysisInfo);
+ }
+ } finally {
+ olapTable.readUnlock();
+ }
}
- public void createTaskForEachColumns(AnalysisInfo jobInfo, Map<Long, BaseAnalysisTask> analysisTasks,
+ private void createTaskForEachColumns(AnalysisInfo jobInfo, Map<Long, BaseAnalysisTask> analysisTasks,
boolean isSync) throws DdlException {
Map<String, Set<String>> columnToPartitions = jobInfo.colToPartitions;
for (Entry<String, Set<String>> entry : columnToPartitions.entrySet()) {
@@ -586,7 +549,7 @@ public class AnalysisManager extends Daemon implements Writable {
}
try {
if (!jobInfo.jobType.equals(JobType.SYSTEM)) {
- replayCreateAnalysisTask(analysisInfo);
+ logCreateAnalysisTask(analysisInfo);
}
} catch (Exception e) {
throw new DdlException("Failed to create analysis task", e);
@@ -606,12 +569,17 @@ public class AnalysisManager extends Daemon implements Writable {
Env.getCurrentEnv().getEditLog().logCreateAnalysisJob(analysisJob);
}
- @VisibleForTesting
- public void createTableLevelTaskForExternalTable(AnalysisInfo jobInfo,
+ private void createTaskForExternalTable(AnalysisInfo jobInfo,
Map<Long, BaseAnalysisTask> analysisTasks,
boolean isSync) throws DdlException {
-
- if (jobInfo.analysisType == AnalysisType.HISTOGRAM) {
+ TableIf table;
+ try {
+ table = StatisticsUtil.findTable(jobInfo.catalogName, jobInfo.dbName, jobInfo.tblName);
+ } catch (Throwable e) {
+ LOG.warn(e.getMessage());
+ return;
+ }
+ if (jobInfo.analysisType == AnalysisType.HISTOGRAM || !(table instanceof ExternalTable)) {
return;
}
AnalysisInfoBuilder colTaskInfoBuilder = new AnalysisInfoBuilder(jobInfo);
@@ -625,49 +593,120 @@ public class AnalysisManager extends Daemon implements Writable {
return;
}
try {
- replayCreateAnalysisTask(analysisInfo);
+ logCreateAnalysisTask(analysisInfo);
} catch (Exception e) {
throw new DdlException("Failed to create analysis task", e);
}
}
public void updateTaskStatus(AnalysisInfo info, AnalysisState taskState, String message, long time) {
- TaskStatusWrapper taskStatusWrapper = new TaskStatusWrapper(info, taskState, message, time);
- updaters[info.jobType.ordinal()].apply(taskStatusWrapper);
+ if (analysisJobIdToTaskMap.get(info.jobId) == null) {
+ return;
+ }
+ info.state = taskState;
+ info.message = message;
+ // Update the task cost time when task finished or failed. And only log the final state.
+ if (taskState.equals(AnalysisState.FINISHED) || taskState.equals(AnalysisState.FAILED)) {
+ info.timeCostInMs = time - info.lastExecTimeInMs;
+ info.lastExecTimeInMs = time;
+ logCreateAnalysisTask(info);
+ }
+ info.lastExecTimeInMs = time;
+ AnalysisInfo job = analysisJobInfoMap.get(info.jobId);
+ // Synchronize the job state change in job level.
+ synchronized (job) {
+ job.lastExecTimeInMs = time;
+ // Set the job state to RUNNING when its first task becomes RUNNING.
+ if (info.state.equals(AnalysisState.RUNNING) && job.state.equals(AnalysisState.PENDING)) {
+ job.state = AnalysisState.RUNNING;
+ replayCreateAnalysisJob(job);
+ }
+ boolean allFinished = true;
+ boolean hasFailure = false;
+ for (BaseAnalysisTask task : analysisJobIdToTaskMap.get(info.jobId).values()) {
+ AnalysisInfo taskInfo = task.info;
+ if (taskInfo.state.equals(AnalysisState.RUNNING) || taskInfo.state.equals(AnalysisState.PENDING)) {
+ allFinished = false;
+ break;
+ }
+ if (taskInfo.state.equals(AnalysisState.FAILED)) {
+ hasFailure = true;
+ }
+ }
+ if (allFinished) {
+ if (hasFailure) {
+ job.state = AnalysisState.FAILED;
+ logCreateAnalysisJob(job);
+ } else {
+ job.state = AnalysisState.FINISHED;
+ if (job.jobType.equals(JobType.SYSTEM)) {
+ try {
+ updateTableStats(job);
+ } catch (Throwable e) {
+ LOG.warn("Failed to update Table statistics in job: {}", info.toString(), e);
+ }
+ }
+ logCreateAnalysisJob(job);
+ }
+ analysisJobIdToTaskMap.remove(job.jobId);
+ }
+ }
}
- @VisibleForTesting
- public void updateTableStats(AnalysisInfo jobInfo) {
+ private void updateTableStats(AnalysisInfo jobInfo) throws Throwable {
+ Map<String, String> params = buildTableStatsParams(jobInfo);
TableIf tbl = StatisticsUtil.findTable(jobInfo.catalogName,
jobInfo.dbName, jobInfo.tblName);
- // External Table update table stats after table level task finished.
- if (tbl instanceof ExternalTable) {
- return;
- }
- TableStatsMeta tableStats = findTableStatsStatus(tbl.getId());
- if (tableStats == null) {
- updateTableStatsStatus(new TableStatsMeta(tbl.getId(), tbl.estimatedRowCount(), jobInfo));
- } else {
- tableStats.updateByJob(jobInfo);
- logCreateTableStats(tableStats);
+
+ // update olap table stats
+ if (tbl.getType() == TableType.OLAP) {
+ OlapTable table = (OlapTable) tbl;
+ updateOlapTableStats(table, params);
}
+ // External Table doesn't collect table stats here.
+ // We create task for external table to collect table/partition level statistics.
}
- public List<AnalysisInfo> showAnalysisJob(ShowAnalyzeStmt stmt) {
- if (stmt.isAuto()) {
- // It's ok to sync on this field, it would only be assigned when instance init or do checkpoint
- synchronized (autoJobs) {
- return findShowAnalyzeResult(autoJobs, stmt);
- }
+ @SuppressWarnings("rawtypes")
+ private Map<String, String> buildTableStatsParams(AnalysisInfo jobInfo) throws Throwable {
+ CatalogIf catalog = StatisticsUtil.findCatalog(jobInfo.catalogName);
+ DatabaseIf db = StatisticsUtil.findDatabase(jobInfo.catalogName, jobInfo.dbName);
+ TableIf tbl = StatisticsUtil.findTable(jobInfo.catalogName, jobInfo.dbName, jobInfo.tblName);
+ String indexId = String.valueOf(jobInfo.indexId);
+ String id = StatisticsUtil.constructId(tbl.getId(), indexId);
+ Map<String, String> commonParams = new HashMap<>();
+ commonParams.put("id", id);
+ commonParams.put("catalogId", String.valueOf(catalog.getId()));
+ commonParams.put("dbId", String.valueOf(db.getId()));
+ commonParams.put("tblId", String.valueOf(tbl.getId()));
+ commonParams.put("indexId", indexId);
+ commonParams.put("lastAnalyzeTimeInMs", String.valueOf(System.currentTimeMillis()));
+ return commonParams;
+ }
+
+ private void updateOlapTableStats(OlapTable table, Map<String, String> params) throws Throwable {
+ for (Partition partition : table.getPartitions()) {
+ HashMap<String, String> partParams = Maps.newHashMap(params);
+ long rowCount = partition.getBaseIndex().getRowCount();
+ partParams.put("id", StatisticsUtil
+ .constructId(params.get("id"), partition.getId()));
+ partParams.put("partId", String.valueOf(partition.getId()));
+ partParams.put("rowCount", String.valueOf(rowCount));
+ StatisticsRepository.persistTableStats(partParams);
}
- return findShowAnalyzeResult(analysisJobInfoMap.values(), stmt);
+
+ HashMap<String, String> tblParams = Maps.newHashMap(params);
+ long rowCount = table.getRowCount();
+ tblParams.put("partId", "NULL");
+ tblParams.put("rowCount", String.valueOf(rowCount));
+ StatisticsRepository.persistTableStats(tblParams);
}
- protected List<AnalysisInfo> findShowAnalyzeResult(Collection<AnalysisInfo> analysisInfos, ShowAnalyzeStmt stmt) {
+ public List<AnalysisInfo> showAnalysisJob(ShowAnalyzeStmt stmt) {
String state = stmt.getStateValue();
TableName tblName = stmt.getDbTableName();
- return analysisInfos.stream()
+ return analysisJobInfoMap.values().stream()
.filter(a -> stmt.getJobId() == 0 || a.jobId == stmt.getJobId())
.filter(a -> state == null || a.state.equals(AnalysisState.valueOf(state)))
.filter(a -> tblName == null || a.catalogName.equals(tblName.getCtl())
@@ -698,11 +737,10 @@ public class AnalysisManager extends Daemon implements Writable {
break;
}
}
- return String.format(progressDisplayTemplate, finished, failed, inProgress, total);
+ return String.format("%d Finished/%d Failed/%d In Progress/%d Total", finished, failed, inProgress, total);
}
- @VisibleForTesting
- public void syncExecute(Collection<BaseAnalysisTask> tasks) {
+ private void syncExecute(Collection<BaseAnalysisTask> tasks) {
SyncTaskCollection syncTaskCollection = new SyncTaskCollection(tasks);
ConnectContext ctx = ConnectContext.get();
try {
@@ -716,8 +754,7 @@ public class AnalysisManager extends Daemon implements Writable {
private ThreadPoolExecutor createThreadPoolForSyncAnalyze() {
String poolName = "SYNC ANALYZE THREAD POOL";
- return new ThreadPoolExecutor(0,
- ConnectContext.get().getSessionVariable().parallelSyncAnalyzeTaskNum,
+ return new ThreadPoolExecutor(0, 64,
0, TimeUnit.SECONDS,
new SynchronousQueue(),
new ThreadFactoryBuilder().setDaemon(true).setNameFormat("SYNC ANALYZE" + "-%d")
@@ -730,23 +767,17 @@ public class AnalysisManager extends Daemon implements Writable {
Env.getCurrentEnv().getStatisticsCleaner().clear();
return;
}
-
Set<String> cols = dropStatsStmt.getColumnNames();
long tblId = dropStatsStmt.getTblId();
- TableStatsMeta tableStats = findTableStatsStatus(dropStatsStmt.getTblId());
- if (tableStats == null) {
- return;
+ StatisticsRepository.dropStatistics(tblId, cols);
+ for (String col : cols) {
+ Env.getCurrentEnv().getStatisticsCache().invalidate(tblId, -1L, col);
}
- if (cols == null) {
- tableStats.reset();
- } else {
- dropStatsStmt.getColumnNames().forEach(tableStats::removeColumn);
- for (String col : cols) {
- Env.getCurrentEnv().getStatisticsCache().invalidate(tblId, -1L, col);
- }
+ if (dropStatsStmt.dropTableRowCount()) {
+ StatisticsRepository.dropExternalTableStatistics(tblId);
+ // Table cache key doesn't care about catalog id and db id, because the table id is globally unique.
+ Env.getCurrentEnv().getStatisticsCache().invalidateTableStats(-1, -1, tblId);
}
- logCreateTableStats(tableStats);
- StatisticsRepository.dropStatistics(tblId, cols);
}
public void handleKillAnalyzeStmt(KillAnalysisJobStmt killAnalysisJobStmt) throws DdlException {
@@ -841,15 +872,15 @@ public class AnalysisManager extends Daemon implements Writable {
executor.submit(() -> {
try {
if (cancelled) {
- errorMessages.add("Query timeout or user cancelled."
- + "Could set analyze_timeout to a bigger value.");
return;
}
try {
task.execute();
+ updateSyncTaskStatus(task, AnalysisState.FINISHED);
} catch (Throwable t) {
colNames.add(task.info.colName);
errorMessages.add(Util.getRootCauseMessage(t));
+ updateSyncTaskStatus(task, AnalysisState.FAILED);
LOG.warn("Failed to analyze, info: {}", task, t);
}
} finally {
@@ -867,22 +898,32 @@ public class AnalysisManager extends Daemon implements Writable {
+ "] Reasons: " + String.join(",", errorMessages));
}
}
+
+ private void updateSyncTaskStatus(BaseAnalysisTask task, AnalysisState state) {
+ Env.getCurrentEnv().getAnalysisManager()
+ .updateTaskStatus(task.info, state, "", System.currentTimeMillis());
+ }
+ }
+
+ public List<AnalysisInfo> findAutomaticAnalysisJobs() {
+ synchronized (analysisJobInfoMap) {
+ return analysisJobInfoMap.values().stream()
+ .filter(a ->
+ a.scheduleType.equals(ScheduleType.AUTOMATIC)
+ && (!(a.state.equals(AnalysisState.RUNNING)
+ || a.state.equals(AnalysisState.PENDING)))
+ && System.currentTimeMillis() - a.lastExecTimeInMs
+ > TimeUnit.MINUTES.toMillis(Config.auto_check_statistics_in_minutes))
+ .collect(Collectors.toList());
+ }
}
public List<AnalysisInfo> findPeriodicJobs() {
synchronized (analysisJobInfoMap) {
- Predicate<AnalysisInfo> p = a -> {
- if (a.state.equals(AnalysisState.RUNNING)) {
- return false;
- }
- if (a.cronExpression == null) {
- return a.scheduleType.equals(ScheduleType.PERIOD)
- && System.currentTimeMillis() - a.lastExecTimeInMs > a.periodTimeInMs;
- }
- return a.cronExpression.getTimeAfter(new Date(a.lastExecTimeInMs)).before(new Date());
- };
return analysisJobInfoMap.values().stream()
- .filter(p)
+ .filter(a -> a.scheduleType.equals(ScheduleType.PERIOD)
+ && (a.state.equals(AnalysisState.FINISHED))
+ && System.currentTimeMillis() - a.lastExecTimeInMs > a.periodTimeInMs)
.collect(Collectors.toList());
}
}
@@ -896,8 +937,7 @@ public class AnalysisManager extends Daemon implements Writable {
public List<AnalysisInfo> findTasksByTaskIds(long jobId) {
AnalysisInfo jobInfo = analysisJobInfoMap.get(jobId);
if (jobInfo != null && jobInfo.taskIds != null) {
- return jobInfo.taskIds.stream().map(analysisTaskInfoMap::get).filter(i -> i != null)
- .collect(Collectors.toList());
+ return jobInfo.taskIds.stream().map(id -> analysisTaskInfoMap.get(id)).collect(Collectors.toList());
}
return null;
}
@@ -923,185 +963,34 @@ public class AnalysisManager extends Daemon implements Writable {
public static AnalysisManager readFields(DataInput in) throws IOException {
AnalysisManager analysisManager = new AnalysisManager();
- readAnalysisInfo(in, analysisManager.analysisJobInfoMap, true);
- readAnalysisInfo(in, analysisManager.analysisTaskInfoMap, false);
- readIdToTblStats(in, analysisManager.idToTblStats);
- readAutoJobs(in, analysisManager);
+ doRead(in, analysisManager.analysisJobInfoMap, true);
+ doRead(in, analysisManager.analysisTaskInfoMap, false);
return analysisManager;
}
- private static void readAnalysisInfo(DataInput in, Map<Long, AnalysisInfo> map, boolean job) throws IOException {
+ private static void doRead(DataInput in, Map<Long, AnalysisInfo> map, boolean job) throws IOException {
int size = in.readInt();
for (int i = 0; i < size; i++) {
AnalysisInfo analysisInfo = AnalysisInfo.read(in);
- // Unfinished manual once job/tasks doesn't need to keep in memory anymore.
- if (needAbandon(analysisInfo)) {
- continue;
- }
map.put(job ? analysisInfo.jobId : analysisInfo.taskId, analysisInfo);
}
}
- // Need to abandon the unfinished manual once jobs/tasks while loading image and replay journal.
- // Journal only store finished tasks and jobs.
- public static boolean needAbandon(AnalysisInfo analysisInfo) {
- if (analysisInfo == null) {
- return true;
- }
- if ((AnalysisState.PENDING.equals(analysisInfo.state) || AnalysisState.RUNNING.equals(analysisInfo.state))
- && ScheduleType.ONCE.equals(analysisInfo.scheduleType)
- && JobType.MANUAL.equals(analysisInfo.jobType)) {
- return true;
- }
- return false;
- }
-
- private static void readIdToTblStats(DataInput in, Map<Long, TableStatsMeta> map) throws IOException {
- int size = in.readInt();
- for (int i = 0; i < size; i++) {
- TableStatsMeta tableStats = TableStatsMeta.read(in);
- map.put(tableStats.tblId, tableStats);
- }
- }
-
- private static void readAutoJobs(DataInput in, AnalysisManager analysisManager) throws IOException {
- Type type = new TypeToken<LinkedList<AnalysisInfo>>() {}.getType();
- Collection<AnalysisInfo> autoJobs = GsonUtils.GSON.fromJson(Text.readString(in), type);
- analysisManager.autoJobs = analysisManager.createSimpleQueue(autoJobs, analysisManager);
- }
-
@Override
public void write(DataOutput out) throws IOException {
- writeJobInfo(out, analysisJobInfoMap);
- writeJobInfo(out, analysisTaskInfoMap);
- writeTableStats(out);
- writeAutoJobsStatus(out);
+ doWrite(out, analysisJobInfoMap);
+ doWrite(out, analysisTaskInfoMap);
}
- private void writeJobInfo(DataOutput out, Map<Long, AnalysisInfo> infoMap) throws IOException {
+ private void doWrite(DataOutput out, Map<Long, AnalysisInfo> infoMap) throws IOException {
out.writeInt(infoMap.size());
for (Entry<Long, AnalysisInfo> entry : infoMap.entrySet()) {
entry.getValue().write(out);
}
}
- private void writeTableStats(DataOutput out) throws IOException {
- out.writeInt(idToTblStats.size());
- for (Entry<Long, TableStatsMeta> entry : idToTblStats.entrySet()) {
- entry.getValue().write(out);
- }
- }
-
- private void writeAutoJobsStatus(DataOutput output) throws IOException {
- Type type = new TypeToken<LinkedList<AnalysisInfo>>() {}.getType();
- String autoJobs = GsonUtils.GSON.toJson(this.autoJobs, type);
- Text.writeString(output, autoJobs);
- }
-
// For unit test use only.
public void addToJobIdTasksMap(long jobId, Map<Long, BaseAnalysisTask> tasks) {
analysisJobIdToTaskMap.put(jobId, tasks);
}
-
- public TableStatsMeta findTableStatsStatus(long tblId) {
- return idToTblStats.get(tblId);
- }
-
- // Invoke this when load transaction finished.
- public void updateUpdatedRows(long tblId, long rows) {
- TableStatsMeta statsStatus = idToTblStats.get(tblId);
- if (statsStatus != null) {
- statsStatus.updatedRows.addAndGet(rows);
- logCreateTableStats(statsStatus);
- }
- }
-
- public void updateTableStatsStatus(TableStatsMeta tableStats) {
- replayUpdateTableStatsStatus(tableStats);
- logCreateTableStats(tableStats);
- }
-
- public void replayUpdateTableStatsStatus(TableStatsMeta tableStats) {
- idToTblStats.put(tableStats.tblId, tableStats);
- }
-
- public void logCreateTableStats(TableStatsMeta tableStats) {
- Env.getCurrentEnv().getEditLog().logCreateTableStats(tableStats);
- }
-
- public void registerSysJob(AnalysisInfo jobInfo, Map<Long, BaseAnalysisTask> taskInfos) {
- jobInfo.state = AnalysisState.RUNNING;
- systemJobInfoMap.put(jobInfo.jobId, jobInfo);
- analysisJobIdToTaskMap.put(jobInfo.jobId, taskInfos);
- }
-
- @VisibleForTesting
- protected Set<String> findReAnalyzeNeededPartitions(TableIf table) {
- TableStatsMeta tableStats = findTableStatsStatus(table.getId());
- if (tableStats == null) {
- return table.getPartitionNames().stream().map(table::getPartition)
- .filter(Partition::hasData).map(Partition::getName).collect(Collectors.toSet());
- }
- return table.getPartitionNames().stream()
- .map(table::getPartition)
- .filter(Partition::hasData)
- .filter(partition ->
- partition.getVisibleVersionTime() >= tableStats.updatedTime).map(Partition::getName)
- .collect(Collectors.toSet());
- }
-
- protected void logAutoJob(AnalysisInfo autoJob) {
- Env.getCurrentEnv().getEditLog().logAutoJob(autoJob);
- }
-
- public void replayPersistSysJob(AnalysisInfo analysisInfo) {
- autoJobs.offer(analysisInfo);
- }
-
- protected SimpleQueue<AnalysisInfo> createSimpleQueue(Collection<AnalysisInfo> collection,
- AnalysisManager analysisManager) {
- return new SimpleQueue<>(Config.auto_analyze_job_record_count,
- a -> {
- // FE is not ready when replaying log and operations triggered by replaying
- // shouldn't be logged again.
- if (Env.getCurrentEnv().isReady() && Env.getCurrentEnv().isMaster() && !Env.isCheckpointThread()) {
- analysisManager.logAutoJob(a);
- }
- return null;
- },
- a -> {
- // DO NOTHING
- return null;
- }, null);
- }
-
- // Remove col stats status from TableStats if failed load some col stats after analyze corresponding column so that
- // we could make sure it would be analyzed again soon if user or system submit job for that column again.
- public void removeColStatsStatus(long tblId, String colName) {
- TableStatsMeta tableStats = findTableStatsStatus(tblId);
- if (tableStats != null) {
- tableStats.removeColumn(colName);
- }
- }
-
- public void removeTableStats(long tblId) {
- if (!idToTblStats.containsKey(tblId)) {
- return;
- }
- TableStatsDeletionLog log = new TableStatsDeletionLog(tblId);
- Env.getCurrentEnv().getEditLog().logDeleteTableStats(log);
- replayTableStatsDeletion(log);
- }
-
- public void replayTableStatsDeletion(TableStatsDeletionLog log) {
- idToTblStats.remove(log.id);
- }
-
- public ColStatsMeta findColStatsMeta(long tblId, String colName) {
- TableStatsMeta tableStats = findTableStatsStatus(tblId);
- if (tableStats == null) {
- return null;
- }
- return tableStats.findColumnStatsMeta(colName);
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisState.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisState.java
index 3abc4c224fa..bab8a462e8a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisState.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisState.java
@@ -18,9 +18,7 @@
package org.apache.doris.statistics;
public enum AnalysisState {
- // When analyze job/task created, but never run
PENDING,
- // When analyze job/task is in running queue
RUNNING,
FINISHED,
FAILED;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskExecutor.java
index 4b133ce0ebf..b5ec7aeb876 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskExecutor.java
@@ -17,7 +17,6 @@
package org.apache.doris.statistics;
-import org.apache.doris.catalog.Env;
import org.apache.doris.common.Config;
import org.apache.doris.common.ThreadPoolManager;
import org.apache.doris.common.ThreadPoolManager.BlockedPolicy;
@@ -36,30 +35,26 @@ public class AnalysisTaskExecutor extends Thread {
private static final Logger LOG = LogManager.getLogger(AnalysisTaskExecutor.class);
- private final ThreadPoolExecutor executors;
+ private final ThreadPoolExecutor executors = ThreadPoolManager.newDaemonThreadPool(
+ Config.statistics_simultaneously_running_task_num,
+ Config.statistics_simultaneously_running_task_num, 0,
+ TimeUnit.DAYS, new LinkedBlockingQueue<>(),
+ new BlockedPolicy("Analysis Job Executor", Integer.MAX_VALUE),
+ "Analysis Job Executor", true);
+
+ private final AnalysisTaskScheduler taskScheduler;
private final BlockingQueue<AnalysisTaskWrapper> taskQueue =
new PriorityBlockingQueue<AnalysisTaskWrapper>(20,
Comparator.comparingLong(AnalysisTaskWrapper::getStartTime));
- public AnalysisTaskExecutor(int simultaneouslyRunningTaskNum) {
- if (!Env.isCheckpointThread()) {
- executors = ThreadPoolManager.newDaemonThreadPool(
- simultaneouslyRunningTaskNum,
- simultaneouslyRunningTaskNum, 0,
- TimeUnit.DAYS, new LinkedBlockingQueue<>(),
- new BlockedPolicy("Analysis Job Executor", Integer.MAX_VALUE),
- "Analysis Job Executor", true);
- } else {
- executors = null;
- }
+ public AnalysisTaskExecutor(AnalysisTaskScheduler jobExecutor) {
+ this.taskScheduler = jobExecutor;
}
@Override
public void run() {
- if (Env.isCheckpointThread()) {
- return;
- }
+ fetchAndExecute();
cancelExpiredTask();
}
@@ -87,7 +82,22 @@ public class AnalysisTaskExecutor extends Thread {
}
}
- public void submitTask(BaseAnalysisTask task) {
+ public void fetchAndExecute() {
+ Thread t = new Thread(() -> {
+ for (;;) {
+ try {
+ doFetchAndExecute();
+ } catch (Throwable throwable) {
+ LOG.warn(throwable);
+ }
+ }
+ }, "Analysis Task Submitter");
+ t.setDaemon(true);
+ t.start();
+ }
+
+ private void doFetchAndExecute() {
+ BaseAnalysisTask task = taskScheduler.getPendingTasks();
AnalysisTaskWrapper taskWrapper = new AnalysisTaskWrapper(this, task);
executors.submit(taskWrapper);
}
@@ -95,13 +105,4 @@ public class AnalysisTaskExecutor extends Thread {
public void putJob(AnalysisTaskWrapper wrapper) throws Exception {
taskQueue.put(wrapper);
}
-
- public boolean idle() {
- return executors.getQueue().isEmpty();
- }
-
- public void clear() {
- executors.getQueue().clear();
- taskQueue.clear();
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskScheduler.java
new file mode 100644
index 00000000000..5c9de2b58b2
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskScheduler.java
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.statistics;
+
+import org.apache.doris.catalog.Env;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.PriorityQueue;
+import java.util.Queue;
+import java.util.Set;
+
+public class AnalysisTaskScheduler {
+
+ private static final Logger LOG = LogManager.getLogger(AnalysisTaskScheduler.class);
+
+ private final PriorityQueue<BaseAnalysisTask> systemJobQueue =
+ new PriorityQueue<>(Comparator.comparingLong(BaseAnalysisTask::getLastExecTime));
+
+ private final Queue<BaseAnalysisTask> manualJobQueue = new LinkedList<>();
+
+ private final Set<BaseAnalysisTask> systemJobSet = new HashSet<>();
+
+ private final Set<BaseAnalysisTask> manualJobSet = new HashSet<>();
+
+ public synchronized void schedule(BaseAnalysisTask analysisTask) {
+ try {
+
+ switch (analysisTask.info.jobType) {
+ case MANUAL:
+ addToManualJobQueue(analysisTask);
+ break;
+ case SYSTEM:
+ addToSystemQueue(analysisTask);
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown job type: " + analysisTask.info.jobType);
+ }
+ } catch (Throwable t) {
+ Env.getCurrentEnv().getAnalysisManager().updateTaskStatus(
+ analysisTask.info, AnalysisState.FAILED, t.getMessage(), System.currentTimeMillis());
+ }
+ }
+
+ // Make sure invoker of this method is synchronized on object.
+
+ private void addToSystemQueue(BaseAnalysisTask analysisJobInfo) {
+ if (systemJobSet.contains(analysisJobInfo)) {
+ return;
+ }
+ systemJobSet.add(analysisJobInfo);
+ systemJobQueue.add(analysisJobInfo);
+ notify();
+ }
+
+ // Make sure invoker of this method is synchronized on object.
+ private void addToManualJobQueue(BaseAnalysisTask analysisJobInfo) {
+ if (manualJobSet.contains(analysisJobInfo)) {
+ return;
+ }
+ manualJobSet.add(analysisJobInfo);
+ manualJobQueue.add(analysisJobInfo);
+ notify();
+ }
+
+ public synchronized BaseAnalysisTask getPendingTasks() {
+ while (true) {
+ if (!manualJobQueue.isEmpty()) {
+ return pollAndRemove(manualJobQueue, manualJobSet);
+ }
+ if (!systemJobQueue.isEmpty()) {
+ return pollAndRemove(systemJobQueue, systemJobSet);
+ }
+ try {
+ wait();
+ } catch (Exception e) {
+ LOG.warn("Thread get interrupted when waiting for pending jobs", e);
+ return null;
+ }
+ }
+ }
+
+ // Poll from queue, remove from set. Make sure invoker of this method is synchronized on object.
+ private BaseAnalysisTask pollAndRemove(Queue<BaseAnalysisTask> q, Set<BaseAnalysisTask> s) {
+ BaseAnalysisTask t = q.poll();
+ s.remove(t);
+ return t;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskWrapper.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskWrapper.java
index 9aa3d85992b..7f55469f533 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskWrapper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskWrapper.java
@@ -18,15 +18,11 @@
package org.apache.doris.statistics;
import org.apache.doris.catalog.Env;
-import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.common.util.Util;
-import org.apache.doris.statistics.AnalysisInfo.ScheduleType;
-import org.apache.doris.statistics.util.StatisticsUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import java.time.LocalTime;
import java.util.concurrent.FutureTask;
public class AnalysisTaskWrapper extends FutureTask<Void> {
@@ -56,14 +52,6 @@ public class AnalysisTaskWrapper extends FutureTask<Void> {
if (task.killed) {
return;
}
- if (task.info.scheduleType.equals(ScheduleType.AUTOMATIC) && !StatisticsUtil.inAnalyzeTime(
- LocalTime.now(TimeUtils.getTimeZone().toZoneId()))) {
- // TODO: Do we need a separate AnalysisState here?
- Env.getCurrentEnv().getAnalysisManager()
- .updateTaskStatus(task.info, AnalysisState.FAILED, "Auto task"
- + "doesn't get executed within specified time range", System.currentTimeMillis());
- return;
- }
executor.putJob(this);
super.run();
Object result = get();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
index fd99c97e83b..719df8769a4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
@@ -21,11 +21,7 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.TableIf;
-import org.apache.doris.common.Config;
import org.apache.doris.datasource.CatalogIf;
-import org.apache.doris.qe.AuditLogHelper;
-import org.apache.doris.qe.QueryState;
-import org.apache.doris.qe.QueryState.MysqlStateType;
import org.apache.doris.qe.StmtExecutor;
import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod;
import org.apache.doris.statistics.AnalysisInfo.AnalysisType;
@@ -68,7 +64,7 @@ public abstract class BaseAnalysisTask {
protected static final String INSERT_COL_STATISTICS = "INSERT INTO "
+ "${internalDB}.${columnStatTbl}"
+ " SELECT id, catalog_id, db_id, tbl_id, idx_id, col_id, part_id, row_count, "
- + " ndv, null_count, CAST(min AS string), CAST(max AS string), data_size, update_time\n"
+ + " ndv, null_count, min, max, data_size, update_time\n"
+ " FROM \n"
+ " (SELECT CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}') AS id, "
+ " ${catalogId} AS catalog_id, "
@@ -115,7 +111,7 @@ public abstract class BaseAnalysisTask {
init(info);
}
- protected void init(AnalysisInfo info) {
+ private void init(AnalysisInfo info) {
catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(info.catalogName);
if (catalog == null) {
Env.getCurrentEnv().getAnalysisManager().updateTaskStatus(info, AnalysisState.FAILED,
@@ -170,9 +166,6 @@ public abstract class BaseAnalysisTask {
doExecute();
break;
} catch (Throwable t) {
- if (killed) {
- throw new RuntimeException(t);
- }
LOG.warn("Failed to execute analysis task, retried times: {}", retriedTimes++, t);
if (retriedTimes > StatisticConstants.ANALYZE_TASK_RETRY_TIMES) {
throw new RuntimeException(t);
@@ -188,16 +181,12 @@ public abstract class BaseAnalysisTask {
if (killed) {
return;
}
- long tblId = tbl.getId();
- String colName = col.getName();
- if (!Env.getCurrentEnv().getStatisticsCache().syncLoadColStats(tblId, -1, colName)) {
- Env.getCurrentEnv().getAnalysisManager().removeColStatsStatus(tblId, colName);
- }
+ Env.getCurrentEnv().getStatisticsCache().syncLoadColStats(tbl.getId(), -1, col.getName());
}
protected void setTaskStateToRunning() {
Env.getCurrentEnv().getAnalysisManager()
- .updateTaskStatus(info, AnalysisState.RUNNING, "", System.currentTimeMillis());
+ .updateTaskStatus(info, AnalysisState.RUNNING, "", System.currentTimeMillis());
}
public void cancel() {
@@ -210,6 +199,10 @@ public abstract class BaseAnalysisTask {
String.format("Job has been cancelled: %s", info.message), System.currentTimeMillis());
}
+ public long getLastExecTime() {
+ return info.lastExecTimeInMs;
+ }
+
public long getJobId() {
return info.jobId;
}
@@ -223,49 +216,21 @@ public abstract class BaseAnalysisTask {
}
protected String getSampleExpression() {
- if (info.forceFull) {
- return "";
- }
- int sampleRows = info.sampleRows;
if (info.analysisMethod == AnalysisMethod.FULL) {
- if (Config.enable_auto_sample
- && tbl.getDataSize(true) > Config.huge_table_lower_bound_size_in_bytes) {
- sampleRows = Config.huge_table_default_sample_rows;
- } else {
- return "";
- }
+ return "";
}
+ // TODO Add sampling methods for external tables
if (info.samplePercent > 0) {
return String.format("TABLESAMPLE(%d PERCENT)", info.samplePercent);
} else {
- return String.format("TABLESAMPLE(%d ROWS)", sampleRows);
+ return String.format("TABLESAMPLE(%d ROWS)", info.sampleRows);
}
}
@Override
public String toString() {
return String.format("Job id [%d], Task id [%d], catalog [%s], db [%s], table [%s], column [%s]",
- info.jobId, info.taskId, catalog.getName(), db.getFullName(), tbl.getName(),
- col == null ? "TableRowCount" : col.getName());
- }
-
- protected void executeWithExceptionOnFail(StmtExecutor stmtExecutor) throws Exception {
- if (killed) {
- return;
- }
- LOG.debug("execute internal sql: {}", stmtExecutor.getOriginStmt());
- try {
- stmtExecutor.execute();
- QueryState queryState = stmtExecutor.getContext().getState();
- if (queryState.getStateType().equals(MysqlStateType.ERR)) {
- throw new RuntimeException(String.format("Failed to analyze %s.%s.%s, error: %s sql: %s",
- info.catalogName, info.dbName, info.colName, stmtExecutor.getOriginStmt().toString(),
- queryState.getErrorMessage()));
- }
- } finally {
- AuditLogHelper.logAuditLog(stmtExecutor.getContext(), stmtExecutor.getOriginStmt().toString(),
- stmtExecutor.getParsedStmt(), stmtExecutor.getQueryStatisticsForAuditLog(),
- true);
- }
+ info.jobId, info.taskId, catalog.getName(), db.getFullName(), tbl.getName(),
+ col == null ? "TableRowCount" : col.getName());
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsData.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsData.java
deleted file mode 100644
index a14e32d726b..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsData.java
+++ /dev/null
@@ -1,82 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.statistics;
-
-import org.apache.doris.statistics.util.StatisticsUtil;
-
-import java.util.StringJoiner;
-
-/**
- * Used to convert data from ResultRow.
- * 0: id
- * 1: catalog_id
- * 2: db_id
- * 3: tbl_id
- * 4: idx_id
- * 5: col_id
- * 6: part_id
- * 7: count
- * 8: ndv
- * 9: null_count
- * 10: min
- * 11: max
- * 12: data_size_in_bytes
- * 13: update_time
- */
-public class ColStatsData {
- public final StatsId statsId;
- public final long count;
- public final long ndv;
-
- public final long nullCount;
-
- public final String minLit;
- public final String maxLit;
-
- public final long dataSizeInBytes;
-
- public final String updateTime;
-
- public ColStatsData(ResultRow row) {
- this.statsId = new StatsId(row);
- this.count = Long.parseLong(row.get(7));
- this.ndv = Long.parseLong(row.getWithDefault(8, "0"));
- this.nullCount = Long.parseLong(row.getWithDefault(9, "0"));
- this.minLit = row.get(10);
- this.maxLit = row.get(11);
- this.dataSizeInBytes = Long.parseLong(row.getWithDefault(12, "0"));
- this.updateTime = row.get(13);
- }
-
- public String toSQL(boolean roundByParentheses) {
- StringJoiner sj = null;
- if (roundByParentheses) {
- sj = new StringJoiner(",", "(" + statsId.toSQL() + ",", ")");
- } else {
- sj = new StringJoiner(",", statsId.toSQL(), "");
- }
- sj.add(String.valueOf(count));
- sj.add(String.valueOf(ndv));
- sj.add(String.valueOf(nullCount));
- sj.add(StatisticsUtil.quote(minLit));
- sj.add(StatisticsUtil.quote(maxLit));
- sj.add(String.valueOf(dataSizeInBytes));
- sj.add(StatisticsUtil.quote(updateTime));
- return sj.toString();
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsMeta.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsMeta.java
deleted file mode 100644
index 445641b2505..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsMeta.java
+++ /dev/null
@@ -1,58 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.statistics;
-
-import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod;
-import org.apache.doris.statistics.AnalysisInfo.AnalysisType;
-import org.apache.doris.statistics.AnalysisInfo.JobType;
-
-import com.google.gson.annotations.SerializedName;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-public class ColStatsMeta {
-
- @SerializedName("updateTime")
- public long updatedTime;
-
- @SerializedName("method")
- public AnalysisMethod analysisMethod;
-
- @SerializedName("type")
- public AnalysisType analysisType;
-
- @SerializedName("queriedTimes")
- public final AtomicLong queriedTimes = new AtomicLong();
-
- // TODO: For column that manually analyzed, we should use same analyze method as user specified.
- @SerializedName("trigger")
- public JobType jobType;
-
- public ColStatsMeta(long updatedTime, AnalysisMethod analysisMethod,
- AnalysisType analysisType, JobType jobType, long queriedTimes) {
- this.updatedTime = updatedTime;
- this.analysisMethod = analysisMethod;
- this.analysisType = analysisType;
- this.jobType = jobType;
- this.queriedTimes.addAndGet(queriedTimes);
- }
-
- public void clear() {
- updatedTime = 0;
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
index c6b019f669b..7986cb07a57 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
@@ -19,26 +19,26 @@ package org.apache.doris.statistics;
import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.catalog.Column;
-import org.apache.doris.catalog.PartitionInfo;
+import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.statistics.util.InternalQueryResult.ResultRow;
import org.apache.doris.statistics.util.StatisticsUtil;
-import com.google.common.collect.Sets;
import com.google.gson.annotations.SerializedName;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.json.JSONObject;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
public class ColumnStatistic {
- public static final double STATS_ERROR = 0.1D;
-
public static final StatsType NDV = StatsType.NDV;
public static final StatsType AVG_SIZE = StatsType.AVG_SIZE;
public static final StatsType MAX_SIZE = StatsType.MAX_SIZE;
@@ -50,17 +50,30 @@ public class ColumnStatistic {
public static ColumnStatistic UNKNOWN = new ColumnStatisticBuilder().setAvgSizeByte(1).setNdv(1)
.setNumNulls(1).setCount(1).setMaxValue(Double.POSITIVE_INFINITY).setMinValue(Double.NEGATIVE_INFINITY)
- .setIsUnknown(true).setUpdatedTime("")
+ .setSelectivity(1.0).setIsUnknown(true)
.build();
public static ColumnStatistic ZERO = new ColumnStatisticBuilder().setAvgSizeByte(0).setNdv(0)
.setNumNulls(0).setCount(0).setMaxValue(Double.NaN).setMinValue(Double.NaN)
+ .setSelectivity(0)
.build();
- public static final Set<Type> UNSUPPORTED_TYPE = Sets.newHashSet(
- Type.HLL, Type.BITMAP, Type.ARRAY, Type.STRUCT, Type.MAP, Type.QUANTILE_STATE, Type.AGG_STATE, Type.JSONB,
- Type.VARIANT, Type.TIME, Type.TIMEV2, Type.LAMBDA_FUNCTION
- );
+ public static final Set<Type> UNSUPPORTED_TYPE = new HashSet<>();
+
+ static {
+ UNSUPPORTED_TYPE.add(Type.HLL);
+ UNSUPPORTED_TYPE.add(Type.BITMAP);
+ UNSUPPORTED_TYPE.add(Type.ARRAY);
+ UNSUPPORTED_TYPE.add(Type.STRUCT);
+ UNSUPPORTED_TYPE.add(Type.MAP);
+ UNSUPPORTED_TYPE.add(Type.QUANTILE_STATE);
+ UNSUPPORTED_TYPE.add(Type.AGG_STATE);
+ UNSUPPORTED_TYPE.add(Type.JSONB);
+ UNSUPPORTED_TYPE.add(Type.VARIANT);
+ UNSUPPORTED_TYPE.add(Type.TIME);
+ UNSUPPORTED_TYPE.add(Type.TIMEV2);
+ UNSUPPORTED_TYPE.add(Type.LAMBDA_FUNCTION);
+ }
@SerializedName("count")
public final double count;
@@ -77,6 +90,19 @@ public class ColumnStatistic {
@SerializedName("maxValue")
public final double maxValue;
public final boolean isUnKnown;
+ /*
+ selectivity of Column T1.A:
+ if T1.A = T2.B is the inner join condition, for a given `b` in B, b in
+ intersection of range(A) and range(B), selectivity means the probability that
+ the equation can be satisfied.
+ We take tpch as example.
+ l_orderkey = o_orderkey and o_orderstatus='o'
+ there are 3 distinct o_orderstatus in orders table. filter o_orderstatus='o' reduces orders table by 1/3
+ because o_orderkey is primary key, thus the o_orderkey.selectivity = 1/3,
+ and after join(l_orderkey = o_orderkey), lineitem is reduced by 1/3.
+ But after filter, other columns' selectivity is still 1.0
+ */
+ public final double selectivity;
/*
originalNdv is the ndv in stats of ScanNode. ndv may be changed after filter or join,
@@ -85,6 +111,7 @@ public class ColumnStatistic {
*/
public final ColumnStatistic original;
+ // For display only.
public final LiteralExpr minExpr;
public final LiteralExpr maxExpr;
@@ -92,17 +119,14 @@ public class ColumnStatistic {
// assign value when do stats estimation.
public final Histogram histogram;
- @SerializedName("partitionIdToColStats")
- public final Map<String, ColumnStatistic> partitionIdToColStats = new HashMap<>();
+ public final Map<Long, ColumnStatistic> partitionIdToColStats = new HashMap<>();
public final String updatedTime;
- public final PartitionInfo partitionInfo;
-
public ColumnStatistic(double count, double ndv, ColumnStatistic original, double avgSizeByte,
double numNulls, double dataSize, double minValue, double maxValue,
- LiteralExpr minExpr, LiteralExpr maxExpr, boolean isUnKnown, Histogram histogram,
- String updatedTime, PartitionInfo partitionInfo) {
+ double selectivity, LiteralExpr minExpr, LiteralExpr maxExpr, boolean isUnKnown, Histogram histogram,
+ String updatedTime) {
this.count = count;
this.ndv = ndv;
this.original = original;
@@ -111,30 +135,31 @@ public class ColumnStatistic {
this.dataSize = dataSize;
this.minValue = minValue;
this.maxValue = maxValue;
+ this.selectivity = selectivity;
this.minExpr = minExpr;
this.maxExpr = maxExpr;
this.isUnKnown = isUnKnown;
this.histogram = histogram;
this.updatedTime = updatedTime;
- this.partitionInfo = partitionInfo;
}
public static ColumnStatistic fromResultRow(List<ResultRow> resultRows) {
- Map<String, ColumnStatistic> partitionIdToColStats = new HashMap<>();
+ Map<Long, ColumnStatistic> partitionIdToColStats = new HashMap<>();
ColumnStatistic columnStatistic = null;
try {
for (ResultRow resultRow : resultRows) {
- String partId = resultRow.get(6);
+ String partId = resultRow.getColumnValue("part_id");
if (partId == null) {
columnStatistic = fromResultRow(resultRow);
} else {
- partitionIdToColStats.put(partId, fromResultRow(resultRow));
+ partitionIdToColStats.put(Long.parseLong(partId), fromResultRow(resultRow));
}
}
} catch (Throwable t) {
LOG.debug("Failed to deserialize column stats", t);
return ColumnStatistic.UNKNOWN;
}
+ // Means last analyze failed or interrupted for some reason.
if (columnStatistic == null) {
return ColumnStatistic.UNKNOWN;
}
@@ -143,44 +168,47 @@ public class ColumnStatistic {
}
// TODO: use thrift
- public static ColumnStatistic fromResultRow(ResultRow row) {
+ public static ColumnStatistic fromResultRow(ResultRow resultRow) {
try {
ColumnStatisticBuilder columnStatisticBuilder = new ColumnStatisticBuilder();
- double count = Double.parseDouble(row.get(7));
+ double count = Double.parseDouble(resultRow.getColumnValueWithDefault("count", "0"));
columnStatisticBuilder.setCount(count);
- double ndv = Double.parseDouble(row.getWithDefault(8, "0"));
+ double ndv = Double.parseDouble(resultRow.getColumnValueWithDefault("ndv", "0"));
+ if (0.99 * count < ndv && ndv < 1.01 * count) {
+ ndv = count;
+ }
columnStatisticBuilder.setNdv(ndv);
- String nullCount = row.getWithDefault(9, "0");
+ String nullCount = resultRow.getColumnValueWithDefault("null_count", "0");
columnStatisticBuilder.setNumNulls(Double.parseDouble(nullCount));
columnStatisticBuilder.setDataSize(Double
- .parseDouble(row.getWithDefault(12, "0")));
+ .parseDouble(resultRow.getColumnValueWithDefault("data_size_in_bytes", "0")));
columnStatisticBuilder.setAvgSizeByte(columnStatisticBuilder.getCount() == 0
? 0 : columnStatisticBuilder.getDataSize()
/ columnStatisticBuilder.getCount());
- long catalogId = Long.parseLong(row.get(1));
- long idxId = Long.parseLong(row.get(4));
- long dbID = Long.parseLong(row.get(2));
- long tblId = Long.parseLong(row.get(3));
- String colName = row.get(5);
+ long catalogId = Long.parseLong(resultRow.getColumnValue("catalog_id"));
+ long idxId = Long.parseLong(resultRow.getColumnValue("idx_id"));
+ long dbID = Long.parseLong(resultRow.getColumnValue("db_id"));
+ long tblId = Long.parseLong(resultRow.getColumnValue("tbl_id"));
+ String colName = resultRow.getColumnValue("col_id");
Column col = StatisticsUtil.findColumn(catalogId, dbID, tblId, idxId, colName);
if (col == null) {
- LOG.debug("Failed to deserialize column statistics, ctlId: {} dbId: {}"
- + "tblId: {} column: {} not exists",
+ LOG.warn("Failed to deserialize column statistics, ctlId: {} dbId: {}"
+ + "tblId: {} column: {} not exists",
catalogId, dbID, tblId, colName);
return ColumnStatistic.UNKNOWN;
}
- String min = row.get(10);
- String max = row.get(11);
+ String min = resultRow.getColumnValue("min");
+ String max = resultRow.getColumnValue("max");
if (min != null && !min.equalsIgnoreCase("NULL")) {
try {
columnStatisticBuilder.setMinValue(StatisticsUtil.convertToDouble(col.getType(), min));
columnStatisticBuilder.setMinExpr(StatisticsUtil.readableValue(col.getType(), min));
} catch (AnalysisException e) {
LOG.warn("Failed to deserialize column {} min value {}.", col, min, e);
- columnStatisticBuilder.setMinValue(Double.NEGATIVE_INFINITY);
+ columnStatisticBuilder.setMinValue(Double.MIN_VALUE);
}
} else {
- columnStatisticBuilder.setMinValue(Double.NEGATIVE_INFINITY);
+ columnStatisticBuilder.setMinValue(Double.MIN_VALUE);
}
if (max != null && !max.equalsIgnoreCase("NULL")) {
try {
@@ -188,12 +216,16 @@ public class ColumnStatistic {
columnStatisticBuilder.setMaxExpr(StatisticsUtil.readableValue(col.getType(), max));
} catch (AnalysisException e) {
LOG.warn("Failed to deserialize column {} max value {}.", col, max, e);
- columnStatisticBuilder.setMaxValue(Double.POSITIVE_INFINITY);
+ columnStatisticBuilder.setMaxValue(Double.MAX_VALUE);
}
} else {
- columnStatisticBuilder.setMaxValue(Double.POSITIVE_INFINITY);
+ columnStatisticBuilder.setMaxValue(Double.MAX_VALUE);
}
- columnStatisticBuilder.setUpdatedTime(row.get(13));
+ columnStatisticBuilder.setSelectivity(1.0);
+ Histogram histogram = Env.getCurrentEnv().getStatisticsCache().getHistogram(tblId, idxId, colName)
+ .orElse(null);
+ columnStatisticBuilder.setHistogram(histogram);
+ columnStatisticBuilder.setUpdatedTime(resultRow.getColumnValue("update_time"));
return columnStatisticBuilder.build();
} catch (Exception e) {
LOG.warn("Failed to deserialize column statistics.", e);
@@ -205,12 +237,25 @@ public class ColumnStatistic {
return rowCount * 0.9 < ndv && ndv < rowCount * 1.1;
}
+ public ColumnStatistic copy() {
+ return new ColumnStatisticBuilder().setCount(count).setNdv(ndv).setAvgSizeByte(avgSizeByte)
+ .setNumNulls(numNulls).setDataSize(dataSize).setMinValue(minValue)
+ .setMaxValue(maxValue).setMinExpr(minExpr).setMaxExpr(maxExpr)
+ .setSelectivity(selectivity).setIsUnknown(isUnKnown).build();
+ }
+
public ColumnStatistic updateByLimit(long limit, double rowCount) {
double ratio = 0;
if (rowCount != 0) {
ratio = limit / rowCount;
}
double newNdv = Math.ceil(Math.min(ndv, limit));
+ double newSelectivity = selectivity;
+ if (newNdv != 0) {
+ newSelectivity = newSelectivity * newNdv / ndv;
+ } else {
+ newSelectivity = 0;
+ }
return new ColumnStatisticBuilder()
.setCount(Math.ceil(limit))
.setNdv(newNdv)
@@ -221,6 +266,7 @@ public class ColumnStatistic {
.setMaxValue(maxValue)
.setMinExpr(minExpr)
.setMaxExpr(maxExpr)
+ .setSelectivity(newSelectivity)
.setIsUnknown(isUnKnown)
.build();
}
@@ -236,11 +282,14 @@ public class ColumnStatistic {
ColumnStatisticBuilder builder = new ColumnStatisticBuilder(this);
Double rowsAfterFilter = rowCount * selectivity;
if (isAlmostUnique(ndv, rowCount)) {
+ builder.setSelectivity(this.selectivity * selectivity);
builder.setNdv(ndv * selectivity);
} else {
if (ndv > rowsAfterFilter) {
+ builder.setSelectivity(this.selectivity * rowsAfterFilter / ndv);
builder.setNdv(rowsAfterFilter);
} else {
+ builder.setSelectivity(this.selectivity);
builder.setNdv(this.ndv);
}
}
@@ -283,8 +332,8 @@ public class ColumnStatistic {
@Override
public String toString() {
- return isUnKnown ? "unknown" : String.format("ndv=%.4f, min=%f(%s), max=%f(%s), count=%.4f, avgSizeByte=%f",
- ndv, minValue, minExpr, maxValue, maxExpr, count, avgSizeByte);
+ return isUnKnown ? "unknown" : String.format("ndv=%.4f, min=%f(%s), max=%f(%s), count=%.4f",
+ ndv, minValue, minExpr, maxValue, maxExpr, count);
}
public JSONObject toJson() {
@@ -306,16 +355,17 @@ public class ColumnStatistic {
statistic.put("MaxValueType", "Normal");
statistic.put("MaxValue", maxValue);
}
+ statistic.put("Selectivity", selectivity);
statistic.put("Count", count);
statistic.put("AvgSizeByte", avgSizeByte);
statistic.put("NumNulls", numNulls);
statistic.put("DataSize", dataSize);
+ statistic.put("Selectivity", selectivity);
statistic.put("MinExpr", minExpr);
statistic.put("MaxExpr", maxExpr);
statistic.put("IsUnKnown", isUnKnown);
statistic.put("Histogram", Histogram.serializeToJson(histogram));
statistic.put("Original", original);
- statistic.put("LastUpdatedTime", updatedTime);
return statistic;
}
@@ -360,11 +410,12 @@ public class ColumnStatistic {
stat.getDouble("DataSize"),
minValue,
maxValue,
+ stat.getDouble("Selectivity"),
null,
null,
stat.getBoolean("IsUnKnown"),
Histogram.deserializeFromJson(stat.getString("Histogram")),
- stat.getString("LastUpdatedTime"), null
+ stat.getString("lastUpdatedTine")
);
}
@@ -392,7 +443,12 @@ public class ColumnStatistic {
return isUnKnown;
}
- public void putPartStats(String partId, ColumnStatistic columnStatistic) {
- this.partitionIdToColStats.put(partId, columnStatistic);
+ public void loadPartitionStats(long tableId, long idxId, String colName) throws DdlException {
+ List<ResultRow> resultRows = StatisticsRepository.loadPartStats(tableId, idxId, colName);
+ for (ResultRow resultRow : resultRows) {
+ String partId = resultRow.getColumnValue("part_id");
+ ColumnStatistic columnStatistic = ColumnStatistic.fromResultRow(resultRow);
+ partitionIdToColStats.put(Long.parseLong(partId), columnStatistic);
+ }
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticBuilder.java
index fa4cf7ebc99..6ca2cc55b79 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticBuilder.java
@@ -18,10 +18,6 @@
package org.apache.doris.statistics;
import org.apache.doris.analysis.LiteralExpr;
-import org.apache.doris.catalog.PartitionInfo;
-
-import java.util.HashMap;
-import java.util.Map;
public class ColumnStatisticBuilder {
private double count;
@@ -31,6 +27,7 @@ public class ColumnStatisticBuilder {
private double dataSize;
private double minValue;
private double maxValue;
+ private double selectivity = 1.0;
private LiteralExpr minExpr;
private LiteralExpr maxExpr;
@@ -40,24 +37,11 @@ public class ColumnStatisticBuilder {
private ColumnStatistic original;
- private Map<String, ColumnStatistic> partitionIdToColStats = new HashMap<>();
-
private String updatedTime;
- private PartitionInfo partitionInfo;
-
public ColumnStatisticBuilder() {
}
- public PartitionInfo getPartitionInfo() {
- return partitionInfo;
- }
-
- public ColumnStatisticBuilder setPartitionInfo(PartitionInfo partitionInfo) {
- this.partitionInfo = partitionInfo;
- return this;
- }
-
public ColumnStatisticBuilder(ColumnStatistic columnStatistic) {
this.count = columnStatistic.count;
this.ndv = columnStatistic.ndv;
@@ -66,14 +50,13 @@ public class ColumnStatisticBuilder {
this.dataSize = columnStatistic.dataSize;
this.minValue = columnStatistic.minValue;
this.maxValue = columnStatistic.maxValue;
+ this.selectivity = columnStatistic.selectivity;
this.minExpr = columnStatistic.minExpr;
this.maxExpr = columnStatistic.maxExpr;
this.isUnknown = columnStatistic.isUnKnown;
this.histogram = columnStatistic.histogram;
this.original = columnStatistic.original;
- this.partitionIdToColStats.putAll(columnStatistic.partitionIdToColStats);
this.updatedTime = columnStatistic.updatedTime;
- this.partitionInfo = columnStatistic.partitionInfo;
}
public ColumnStatisticBuilder setCount(double count) {
@@ -116,6 +99,11 @@ public class ColumnStatisticBuilder {
return this;
}
+ public ColumnStatisticBuilder setSelectivity(double selectivity) {
+ this.selectivity = selectivity;
+ return this;
+ }
+
public ColumnStatisticBuilder setMinExpr(LiteralExpr minExpr) {
this.minExpr = minExpr;
return this;
@@ -159,6 +147,10 @@ public class ColumnStatisticBuilder {
return maxValue;
}
+ public double getSelectivity() {
+ return selectivity;
+ }
+
public LiteralExpr getMinExpr() {
return minExpr;
}
@@ -184,23 +176,18 @@ public class ColumnStatisticBuilder {
return updatedTime;
}
- public ColumnStatisticBuilder setUpdatedTime(String updatedTime) {
+ public void setUpdatedTime(String updatedTime) {
this.updatedTime = updatedTime;
- return this;
}
public ColumnStatistic build() {
dataSize = Math.max((count - numNulls + 1) * avgSizeByte, 0);
- if (original == null && !isUnknown) {
+ if (original == null) {
original = new ColumnStatistic(count, ndv, null, avgSizeByte, numNulls,
- dataSize, minValue, maxValue, minExpr, maxExpr,
- isUnknown, histogram, updatedTime, partitionInfo);
- original.partitionIdToColStats.putAll(partitionIdToColStats);
+ dataSize, minValue, maxValue, selectivity, minExpr, maxExpr, false,
+ histogram, updatedTime);
}
- ColumnStatistic colStats = new ColumnStatistic(count, ndv, original, avgSizeByte, numNulls,
- dataSize, minValue, maxValue, minExpr, maxExpr,
- isUnknown, histogram, updatedTime, partitionInfo);
- colStats.partitionIdToColStats.putAll(partitionIdToColStats);
- return colStats;
+ return new ColumnStatistic(count, ndv, original, avgSizeByte, numNulls,
+ dataSize, minValue, maxValue, selectivity, minExpr, maxExpr, isUnknown, histogram, updatedTime);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java
index 281a0e82502..d94a90b75f0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java
@@ -19,8 +19,7 @@ package org.apache.doris.statistics;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.TableIf;
-import org.apache.doris.common.ThreadPoolManager;
-import org.apache.doris.qe.InternalQueryExecutionException;
+import org.apache.doris.statistics.util.InternalQueryResult.ResultRow;
import org.apache.doris.statistics.util.StatisticsUtil;
import org.apache.logging.log4j.LogManager;
@@ -28,23 +27,16 @@ import org.apache.logging.log4j.Logger;
import java.util.List;
import java.util.Optional;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.ThreadPoolExecutor.DiscardOldestPolicy;
public class ColumnStatisticsCacheLoader extends StatisticsCacheLoader<Optional<ColumnStatistic>> {
private static final Logger LOG = LogManager.getLogger(ColumnStatisticsCacheLoader.class);
- private static final ThreadPoolExecutor singleThreadPool = ThreadPoolManager.newDaemonFixedThreadPool(
- StatisticConstants.RETRY_LOAD_THREAD_POOL_SIZE,
- StatisticConstants.RETRY_LOAD_QUEUE_SIZE, "STATS_RELOAD",
- true,
- new DiscardOldestPolicy());
-
@Override
protected Optional<ColumnStatistic> doLoad(StatisticsCacheKey key) {
// Load from statistics table.
- Optional<ColumnStatistic> columnStatistic = loadFromStatsTable(key);
+ Optional<ColumnStatistic> columnStatistic = loadFromStatsTable(key.tableId,
+ key.idxId, key.colName);
if (columnStatistic.isPresent()) {
return columnStatistic;
}
@@ -60,14 +52,8 @@ public class ColumnStatisticsCacheLoader extends StatisticsCacheLoader<Optional<
return columnStatistic;
}
- private Optional<ColumnStatistic> loadFromStatsTable(StatisticsCacheKey key) {
- List<ResultRow> columnResults = null;
- try {
- columnResults = StatisticsRepository.loadColStats(key.tableId, key.idxId, key.colName);
- } catch (InternalQueryExecutionException e) {
- retryLoad(key);
- return Optional.empty();
- }
+ private Optional<ColumnStatistic> loadFromStatsTable(long tableId, long idxId, String colName) {
+ List<ResultRow> columnResults = StatisticsRepository.loadColStats(tableId, idxId, colName);
ColumnStatistic columnStatistics;
try {
columnStatistics = StatisticsUtil.deserializeToColumnStatistics(columnResults);
@@ -81,42 +67,4 @@ public class ColumnStatisticsCacheLoader extends StatisticsCacheLoader<Optional<
return Optional.of(columnStatistics);
}
}
-
- private void retryLoad(StatisticsCacheKey key) {
- singleThreadPool.submit(new RetryTask(key, 1));
- }
-
- private static class RetryTask implements Runnable {
- StatisticsCacheKey key;
- int retryTimes;
-
- public RetryTask(StatisticsCacheKey key, int retryTimes) {
- this.key = key;
- this.retryTimes = retryTimes;
- }
-
- @Override
- public void run() {
- List<ResultRow> columnResults = null;
- try {
- columnResults = StatisticsRepository.loadColStats(key.tableId, key.idxId, key.colName);
- } catch (InternalQueryExecutionException e) {
- if (this.retryTimes < StatisticConstants.LOAD_RETRY_TIMES) {
- retryTimes++;
- singleThreadPool.submit(this);
- }
- return;
- }
- ColumnStatistic columnStatistics;
- try {
- columnStatistics = StatisticsUtil.deserializeToColumnStatistics(columnResults);
- } catch (Exception e) {
- LOG.warn("Exception to deserialize column statistics", e);
- return;
- }
- if (columnStatistics != null) {
- Env.getCurrentEnv().getStatisticsCache().putCache(key, columnStatistics);
- }
- }
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java
index a446cc2610a..d569cd79bd4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java
@@ -17,28 +17,29 @@
package org.apache.doris.statistics;
-import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.external.HMSExternalTable;
import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.qe.AutoCloseConnectContext;
import org.apache.doris.qe.QueryState;
import org.apache.doris.qe.StmtExecutor;
+import org.apache.doris.statistics.util.InternalQueryResult;
import org.apache.doris.statistics.util.StatisticsUtil;
-import com.google.common.collect.Lists;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.text.StringSubstitutor;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
import java.util.ArrayList;
-import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.StringJoiner;
-import java.util.stream.Collectors;
public class HMSAnalysisTask extends BaseAnalysisTask {
private static final Logger LOG = LogManager.getLogger(HMSAnalysisTask.class);
@@ -48,7 +49,7 @@ public class HMSAnalysisTask extends BaseAnalysisTask {
public static final String NUM_FILES = "numFiles";
public static final String TIMESTAMP = "transient_lastDdlTime";
- private static final String ANALYZE_TABLE_TEMPLATE = "INSERT INTO "
+ private static final String ANALYZE_SQL_TABLE_TEMPLATE = "INSERT INTO "
+ "${internalDB}.${columnStatTbl}"
+ " SELECT "
+ "CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}') AS id, "
@@ -58,17 +59,19 @@ public class HMSAnalysisTask extends BaseAnalysisTask {
+ "${idxId} AS idx_id, "
+ "'${colId}' AS col_id, "
+ "NULL AS part_id, "
- + "${countExpr} AS row_count, "
+ + "COUNT(1) AS row_count, "
+ "NDV(`${colName}`) AS ndv, "
- + "${nullCountExpr} AS null_count, "
+ + "SUM(CASE WHEN `${colName}` IS NULL THEN 1 ELSE 0 END) AS null_count, "
+ "MIN(`${colName}`) AS min, "
+ "MAX(`${colName}`) AS max, "
+ "${dataSizeFunction} AS data_size, "
+ "NOW() "
- + "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${sampleExpr}";
+ + "FROM `${catalogName}`.`${dbName}`.`${tblName}`";
- private static final String ANALYZE_PARTITION_TEMPLATE = " SELECT "
- + "CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}', '-', ${partId}) AS id, "
+ private static final String ANALYZE_SQL_PARTITION_TEMPLATE = "INSERT INTO "
+ + "${internalDB}.${columnStatTbl}"
+ + " SELECT "
+ + "CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}') AS id, "
+ "${catalogId} AS catalog_id, "
+ "${dbId} AS db_id, "
+ "${tblId} AS tbl_id, "
@@ -81,22 +84,22 @@ public class HMSAnalysisTask extends BaseAnalysisTask {
+ "MIN(`${colName}`) AS min, "
+ "MAX(`${colName}`) AS max, "
+ "${dataSizeFunction} AS data_size, "
- + "NOW() FROM `${catalogName}`.`${dbName}`.`${tblName}` where ";
-
- private static final String ANALYZE_TABLE_COUNT_TEMPLATE = "SELECT ${countExpr} as rowCount "
- + "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${sampleExpr}";
+ + "NOW() "
+ + "FROM `${catalogName}`.`${dbName}`.`${tblName}`";
- // cache stats for each partition, it would be inserted into column_statistics in a batch.
- private final List<List<ColStatsData>> buf = new ArrayList<>();
+ private static final String ANALYZE_TABLE_COUNT_TEMPLATE = "SELECT COUNT(1) as rowCount "
+ + "FROM `${catalogName}`.`${dbName}`.`${tblName}`";
private final boolean isTableLevelTask;
+ private final boolean isSamplingPartition;
private final boolean isPartitionOnly;
- private Set<String> partitionNames;
+ private final Set<String> partitionNames;
private HMSExternalTable table;
public HMSAnalysisTask(AnalysisInfo info) {
super(info);
isTableLevelTask = info.externalTableLevelTask;
+ isSamplingPartition = info.samplingPartition;
isPartitionOnly = info.partitionOnly;
partitionNames = info.partitionNames;
table = (HMSExternalTable) tbl;
@@ -111,17 +114,42 @@ public class HMSAnalysisTask extends BaseAnalysisTask {
}
/**
- * Get table row count
+ * Get table row count and insert the result to __internal_schema.table_statistics
*/
private void getTableStats() throws Exception {
- Map<String, String> params = buildTableStatsParams(null);
- List<ResultRow> columnResult =
- StatisticsUtil.execStatisticQuery(new StringSubstitutor(params)
- .replace(ANALYZE_TABLE_COUNT_TEMPLATE));
- String rowCount = columnResult.get(0).get(0);
- Env.getCurrentEnv().getAnalysisManager()
- .updateTableStatsStatus(
- new TableStatsMeta(table.getId(), Long.parseLong(rowCount), info));
+ // Get table level information. An example sql for table stats:
+ // INSERT INTO __internal_schema.table_statistics VALUES
+ // ('13055', 13002, 13038, 13055, -1, 'NULL', 5, 1686111064658, NOW())
+ Map<String, String> parameters = table.getRemoteTable().getParameters();
+ if (isPartitionOnly) {
+ for (String partId : partitionNames) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(ANALYZE_SQL_PARTITION_TEMPLATE);
+ sb.append(" where ");
+ String[] splits = partId.split("/");
+ for (int i = 0; i < splits.length; i++) {
+ String value = splits[i].split("=")[1];
+ splits[i] = splits[i].replace(value, "\'" + value + "\'");
+ }
+ sb.append(StringUtils.join(splits, " and "));
+ Map<String, String> params = buildTableStatsParams(partId);
+ setParameterData(parameters, params);
+ List<InternalQueryResult.ResultRow> columnResult =
+ StatisticsUtil.execStatisticQuery(new StringSubstitutor(params)
+ .replace(sb.toString()));
+ String rowCount = columnResult.get(0).getColumnValue("rowCount");
+ params.put("rowCount", rowCount);
+ StatisticsRepository.persistTableStats(params);
+ }
+ } else {
+ Map<String, String> params = buildTableStatsParams(null);
+ List<InternalQueryResult.ResultRow> columnResult =
+ StatisticsUtil.execStatisticQuery(new StringSubstitutor(params)
+ .replace(ANALYZE_TABLE_COUNT_TEMPLATE));
+ String rowCount = columnResult.get(0).getColumnValue("rowCount");
+ params.put("rowCount", rowCount);
+ StatisticsRepository.persistTableStats(params);
+ }
}
/**
@@ -145,102 +173,67 @@ public class HMSAnalysisTask extends BaseAnalysisTask {
// 0 AS data_size,
// NOW() FROM `hive`.`tpch100`.`region`
if (isPartitionOnly) {
- getPartitionNames();
- List<String> partitionAnalysisSQLs = new ArrayList<>();
- for (String partId : this.partitionNames) {
- partitionAnalysisSQLs.add(generateSqlForPartition(partId));
+ for (String partId : partitionNames) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(ANALYZE_SQL_TABLE_TEMPLATE);
+ sb.append(" where ");
+ String[] splits = partId.split("/");
+ for (int i = 0; i < splits.length; i++) {
+ String value = splits[i].split("=")[1];
+ splits[i] = splits[i].replace(value, "\'" + value + "\'");
+ }
+ sb.append(StringUtils.join(splits, " and "));
+ Map<String, String> params = buildTableStatsParams(partId);
+ params.put("internalDB", FeConstants.INTERNAL_DB_NAME);
+ params.put("columnStatTbl", StatisticConstants.STATISTIC_TBL_NAME);
+ params.put("colName", col.getName());
+ params.put("colId", info.colName);
+ params.put("dataSizeFunction", getDataSizeFunction(col));
+ StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
+ String sql = stringSubstitutor.replace(sb.toString());
+ executeInsertSql(sql);
}
- execSQLs(partitionAnalysisSQLs);
} else {
StringBuilder sb = new StringBuilder();
- sb.append(ANALYZE_TABLE_TEMPLATE);
+ sb.append(ANALYZE_SQL_TABLE_TEMPLATE);
+ if (isSamplingPartition) {
+ sb.append(" where 1=1 ");
+ String[] splitExample = partitionNames.stream().findFirst().get().split("/");
+ int parts = splitExample.length;
+ List<String> partNames = new ArrayList<>();
+ for (String split : splitExample) {
+ partNames.add(split.split("=")[0]);
+ }
+ List<List<String>> valueLists = new ArrayList<>();
+ for (int i = 0; i < parts; i++) {
+ valueLists.add(new ArrayList<>());
+ }
+ for (String partId : partitionNames) {
+ String[] partIds = partId.split("/");
+ for (int i = 0; i < partIds.length; i++) {
+ valueLists.get(i).add("\'" + partIds[i].split("=")[1] + "\'");
+ }
+ }
+ for (int i = 0; i < parts; i++) {
+ sb.append(" and ");
+ sb.append(partNames.get(i));
+ sb.append(" in (");
+ sb.append(StringUtils.join(valueLists.get(i), ","));
+ sb.append(") ");
+ }
+ }
Map<String, String> params = buildTableStatsParams("NULL");
params.put("internalDB", FeConstants.INTERNAL_DB_NAME);
params.put("columnStatTbl", StatisticConstants.STATISTIC_TBL_NAME);
params.put("colName", col.getName());
params.put("colId", info.colName);
params.put("dataSizeFunction", getDataSizeFunction(col));
- params.put("nullCountExpr", getNullCountExpression());
StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
String sql = stringSubstitutor.replace(sb.toString());
executeInsertSql(sql);
}
}
- private void getPartitionNames() {
- if (partitionNames == null) {
- if (info.isAllPartition) {
- partitionNames = table.getPartitionNames();
- } else if (info.partitionCount > 0) {
- partitionNames = table.getPartitionNames().stream()
- .limit(info.partitionCount).collect(Collectors.toSet());
- }
- if (partitionNames == null || partitionNames.isEmpty()) {
- throw new RuntimeException("Not a partition table or no partition specified.");
- }
- }
- }
-
- private String generateSqlForPartition(String partId) {
- StringBuilder sb = new StringBuilder();
- sb.append(ANALYZE_PARTITION_TEMPLATE);
- String[] splits = partId.split("/");
- for (int i = 0; i < splits.length; i++) {
- String[] kv = splits[i].split("=");
- sb.append(kv[0]);
- sb.append("='");
- sb.append(kv[1]);
- sb.append("'");
- if (i < splits.length - 1) {
- sb.append(" and ");
- }
- }
- Map<String, String> params = buildTableStatsParams(partId);
- params.put("internalDB", FeConstants.INTERNAL_DB_NAME);
- params.put("columnStatTbl", StatisticConstants.STATISTIC_TBL_NAME);
- params.put("colName", col.getName());
- params.put("colId", info.colName);
- params.put("dataSizeFunction", getDataSizeFunction(col));
- return new StringSubstitutor(params).replace(sb.toString());
- }
-
- public void execSQLs(List<String> partitionAnalysisSQLs) throws Exception {
- long startTime = System.currentTimeMillis();
- LOG.debug("analyze task {} start at {}", info.toString(), new Date());
- try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext()) {
- List<List<String>> sqlGroups = Lists.partition(partitionAnalysisSQLs, StatisticConstants.UNION_ALL_LIMIT);
- for (List<String> group : sqlGroups) {
- if (killed) {
- return;
- }
- StringJoiner partitionCollectSQL = new StringJoiner("UNION ALL");
- group.forEach(partitionCollectSQL::add);
- stmtExecutor = new StmtExecutor(r.connectContext, partitionCollectSQL.toString());
- buf.add(stmtExecutor.executeInternalQuery()
- .stream().map(ColStatsData::new).collect(Collectors.toList()));
- QueryState queryState = r.connectContext.getState();
- if (queryState.getStateType().equals(QueryState.MysqlStateType.ERR)) {
- throw new RuntimeException(String.format("Failed to analyze %s.%s.%s, error: %s sql: %s",
- info.catalogName, info.dbName, info.colName, partitionCollectSQL,
- queryState.getErrorMessage()));
- }
- }
- for (List<ColStatsData> colStatsDataList : buf) {
- StringBuilder batchInsertSQL =
- new StringBuilder("INSERT INTO " + StatisticConstants.FULL_QUALIFIED_STATS_TBL_NAME
- + " VALUES ");
- StringJoiner sj = new StringJoiner(",");
- colStatsDataList.forEach(c -> sj.add(c.toSQL(true)));
- batchInsertSQL.append(sj);
- stmtExecutor = new StmtExecutor(r.connectContext, batchInsertSQL.toString());
- executeWithExceptionOnFail(stmtExecutor);
- }
- } finally {
- LOG.debug("analyze task {} end. cost {}ms", info, System.currentTimeMillis() - startTime);
- }
-
- }
-
private void executeInsertSql(String sql) throws Exception {
long startTime = System.currentTimeMillis();
try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext()) {
@@ -277,8 +270,6 @@ public class HMSAnalysisTask extends BaseAnalysisTask {
commonParams.put("catalogName", catalog.getName());
commonParams.put("dbName", db.getFullName());
commonParams.put("tblName", tbl.getName());
- commonParams.put("sampleExpr", getSampleExpression());
- commonParams.put("countExpr", getCountExpression());
if (col != null) {
commonParams.put("type", col.getType().toString());
}
@@ -286,39 +277,28 @@ public class HMSAnalysisTask extends BaseAnalysisTask {
return commonParams;
}
- protected String getCountExpression() {
- if (info.samplePercent > 0) {
- return String.format("ROUND(COUNT(1) * 100 / %d)", info.samplePercent);
- } else {
- return "COUNT(1)";
+ private void setParameterData(Map<String, String> parameters, Map<String, String> params) {
+ String numRows = "";
+ String timestamp = "";
+ if (parameters.containsKey(NUM_ROWS)) {
+ numRows = parameters.get(NUM_ROWS);
}
- }
-
- protected String getNullCountExpression() {
- if (info.samplePercent > 0) {
- return String.format("ROUND(SUM(CASE WHEN `${colName}` IS NULL THEN 1 ELSE 0 END) * 100 / %d)",
- info.samplePercent);
- } else {
- return "SUM(CASE WHEN `${colName}` IS NULL THEN 1 ELSE 0 END)";
- }
- }
-
- protected String getDataSizeFunction(Column column) {
- String originFunction = super.getDataSizeFunction(column);
- if (info.samplePercent > 0 && !isPartitionOnly) {
- return String.format("ROUND((%s) * 100 / %d)", originFunction, info.samplePercent);
- } else {
- return originFunction;
+ if (parameters.containsKey(TIMESTAMP)) {
+ timestamp = parameters.get(TIMESTAMP);
}
+ params.put("numRows", numRows);
+ params.put("rowCount", numRows);
+ params.put("update_time", TimeUtils.DATETIME_FORMAT.format(
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(Long.parseLong(timestamp) * 1000),
+ ZoneId.systemDefault())));
}
@Override
protected void afterExecution() {
- // Table level task doesn't need to sync any value to sync stats, it stores the value in metadata.
- // Partition only task doesn't need to refresh cached.
- if (isTableLevelTask || isPartitionOnly) {
- return;
+ if (isTableLevelTask) {
+ Env.getCurrentEnv().getStatisticsCache().refreshTableStatsSync(catalog.getId(), db.getId(), tbl.getId());
+ } else {
+ Env.getCurrentEnv().getStatisticsCache().syncLoadColStats(tbl.getId(), -1, col.getName());
}
- Env.getCurrentEnv().getStatisticsCache().syncLoadColStats(tbl.getId(), -1, col.getName());
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistData.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistData.java
deleted file mode 100644
index 85f2fe45bd5..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistData.java
+++ /dev/null
@@ -1,36 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.statistics;
-
-public class HistData {
-
- public final StatsId statsId;
-
- public final double sampleRate;
-
- public final String buckets;
-
- public final String updateTime;
-
- public HistData(ResultRow row) {
- this.statsId = new StatsId(row);
- this.sampleRate = Double.parseDouble(row.get(7));
- this.buckets = row.get(8);
- this.updateTime = row.get(9);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/Histogram.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/Histogram.java
index 2068c368c40..05e2c199ed9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/Histogram.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/Histogram.java
@@ -20,6 +20,7 @@ package org.apache.doris.statistics;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Type;
+import org.apache.doris.statistics.util.InternalQueryResult.ResultRow;
import org.apache.doris.statistics.util.StatisticsUtil;
import com.google.common.base.Strings;
@@ -61,12 +62,13 @@ public class Histogram {
public static Histogram fromResultRow(ResultRow resultRow) {
try {
HistogramBuilder histogramBuilder = new HistogramBuilder();
- HistData histData = new HistData(resultRow);
- long catalogId = histData.statsId.catalogId;
- long idxId = histData.statsId.idxId;
- long dbId = histData.statsId.dbId;
- long tblId = histData.statsId.tblId;
- String colName = histData.statsId.colId;
+
+ long catalogId = Long.parseLong(resultRow.getColumnValue("catalog_id"));
+ long idxId = Long.parseLong(resultRow.getColumnValue("idx_id"));
+ long dbId = Long.parseLong(resultRow.getColumnValue("db_id"));
+ long tblId = Long.parseLong(resultRow.getColumnValue("tbl_id"));
+
+ String colName = resultRow.getColumnValue("col_id");
Column col = StatisticsUtil.findColumn(catalogId, dbId, tblId, idxId, colName);
if (col == null) {
LOG.warn("Failed to deserialize histogram statistics, ctlId: {} dbId: {}"
@@ -77,10 +79,10 @@ public class Histogram {
Type dataType = col.getType();
histogramBuilder.setDataType(dataType);
- double sampleRate = histData.sampleRate;
+ double sampleRate = Double.parseDouble(resultRow.getColumnValue("sample_rate"));
histogramBuilder.setSampleRate(sampleRate);
- String json = histData.buckets;
+ String json = resultRow.getColumnValue("buckets");
JsonObject jsonObj = JsonParser.parseString(json).getAsJsonObject();
int bucketNum = jsonObj.get("num_buckets").getAsInt();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramCacheLoader.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramCacheLoader.java
index d9928f2a639..0e075240923 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramCacheLoader.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramCacheLoader.java
@@ -18,6 +18,7 @@
package org.apache.doris.statistics;
import org.apache.doris.common.FeConstants;
+import org.apache.doris.statistics.util.InternalQueryResult.ResultRow;
import org.apache.doris.statistics.util.StatisticsUtil;
import org.apache.commons.collections.CollectionUtils;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/JdbcAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/JdbcAnalysisTask.java
index 58be1510b44..25b9db0b2c7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/JdbcAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/JdbcAnalysisTask.java
@@ -23,6 +23,7 @@ import org.apache.doris.common.FeConstants;
import org.apache.doris.qe.AutoCloseConnectContext;
import org.apache.doris.qe.QueryState;
import org.apache.doris.qe.StmtExecutor;
+import org.apache.doris.statistics.util.InternalQueryResult;
import org.apache.doris.statistics.util.StatisticsUtil;
import org.apache.commons.text.StringSubstitutor;
@@ -80,11 +81,11 @@ public class JdbcAnalysisTask extends BaseAnalysisTask {
*/
private void getTableStats() throws Exception {
Map<String, String> params = buildTableStatsParams(null);
- List<ResultRow> columnResult =
+ List<InternalQueryResult.ResultRow> columnResult =
StatisticsUtil.execStatisticQuery(new StringSubstitutor(params).replace(ANALYZE_TABLE_COUNT_TEMPLATE));
- String rowCount = columnResult.get(0).get(0);
- Env.getCurrentEnv().getAnalysisManager()
- .updateTableStatsStatus(new TableStatsMeta(table.getId(), Long.parseLong(rowCount), info));
+ String rowCount = columnResult.get(0).getColumnValue("rowCount");
+ params.put("rowCount", rowCount);
+ StatisticsRepository.persistTableStats(params);
}
/**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisJob.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisJob.java
deleted file mode 100644
index 877a4f5bd09..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisJob.java
+++ /dev/null
@@ -1,56 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.statistics;
-
-import java.util.List;
-
-public class OlapAnalysisJob {
-
-
-
- private List<String> columns;
-
- private static String collectPartionStatsSQLTemplate =
- " SELECT "
- + "CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}', '-', ${partId}) AS id, "
- + "${catalogId} AS catalog_id, "
- + "${dbId} AS db_id, "
- + "${tblId} AS tbl_id, "
- + "${idxId} AS idx_id, "
- + "'${colId}' AS col_id, "
- + "${partId} AS part_id, "
- + "COUNT(1) AS row_count, "
- + "NDV(`${colName}`) AS ndv, "
- + "SUM(CASE WHEN `${colName}` IS NULL THEN 1 ELSE 0 END) AS null_count, "
- + "MIN(`${colName}`) AS min, "
- + "MAX(`${colName}`) AS max, "
- + "${dataSizeFunction} AS data_size, "
- + "NOW() ";
-
-
- protected void beforeExecution() {
- }
-
- public void execute() {
- }
-
- protected void afterExecution() {
-
- }
-
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
index 180ac9d9839..257708de54f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
@@ -23,66 +23,37 @@ import org.apache.doris.qe.AutoCloseConnectContext;
import org.apache.doris.qe.QueryState;
import org.apache.doris.qe.QueryState.MysqlStateType;
import org.apache.doris.qe.StmtExecutor;
-import org.apache.doris.statistics.AnalysisInfo.JobType;
import org.apache.doris.statistics.util.StatisticsUtil;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Lists;
import org.apache.commons.text.StringSubstitutor;
import java.util.ArrayList;
-import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.StringJoiner;
-import java.util.stream.Collectors;
/**
* Each task analyze one column.
*/
public class OlapAnalysisTask extends BaseAnalysisTask {
+ private static final String ANALYZE_PARTITION_SQL_TEMPLATE = INSERT_PART_STATISTICS
+ + "FROM `${dbName}`.`${tblName}` "
+ + "PARTITION ${partName} ${sampleExpr}";
+
// TODO Currently, NDV is computed for the full table; in fact,
// NDV should only be computed for the relevant partition.
private static final String ANALYZE_COLUMN_SQL_TEMPLATE = INSERT_COL_STATISTICS
+ " (SELECT NDV(`${colName}`) AS ndv "
+ " FROM `${dbName}`.`${tblName}` ${sampleExpr}) t2\n";
- private static final String collectPartitionStatsSQLTemplate =
- " SELECT "
- + "CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}', '-', ${partId}) AS id, "
- + "${catalogId} AS catalog_id, "
- + "${dbId} AS db_id, "
- + "${tblId} AS tbl_id, "
- + "${idxId} AS idx_id, "
- + "'${colId}' AS col_id, "
- + "${partId} AS part_id, "
- + "COUNT(1) AS row_count, "
- + "NDV(`${colName}`) AS ndv, "
- + "SUM(CASE WHEN `${colName}` IS NULL THEN 1 ELSE 0 END) AS null_count, "
- + "MIN(`${colName}`) AS min, "
- + "MAX(`${colName}`) AS max, "
- + "${dataSizeFunction} AS data_size, "
- + "NOW() FROM `${dbName}`.`${tblName}` PARTITION ${partitionName} ${sampleExpr}";
-
- // cache stats for each partition, it would be inserted into column_statistics in a batch.
- private final List<List<ColStatsData>> buf = new ArrayList<>();
-
- @VisibleForTesting
- public OlapAnalysisTask() {
- }
-
public OlapAnalysisTask(AnalysisInfo info) {
super(info);
}
public void doExecute() throws Exception {
- Set<String> partitionNames = info.colToPartitions.get(info.colName);
- if (partitionNames.isEmpty()) {
- return;
- }
Map<String, String> params = new HashMap<>();
params.put("internalDB", FeConstants.INTERNAL_DB_NAME);
params.put("columnStatTbl", StatisticConstants.STATISTIC_TBL_NAME);
@@ -99,65 +70,56 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
List<String> partitionAnalysisSQLs = new ArrayList<>();
try {
tbl.readLock();
-
- for (String partitionName : partitionNames) {
- Partition part = tbl.getPartition(partitionName);
+ Set<String> partNames = info.colToPartitions.get(info.colName);
+ for (String partName : partNames) {
+ Partition part = tbl.getPartition(partName);
if (part == null) {
continue;
}
- params.put("partId", String.valueOf(tbl.getPartition(partitionName).getId()));
+ params.put("partId", String.valueOf(tbl.getPartition(partName).getId()));
// Avoid error when get the default partition
- params.put("partitionName", "`" + partitionName + "`");
+ params.put("partName", "`" + partName + "`");
StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
- partitionAnalysisSQLs.add(stringSubstitutor.replace(collectPartitionStatsSQLTemplate));
+ partitionAnalysisSQLs.add(stringSubstitutor.replace(ANALYZE_PARTITION_SQL_TEMPLATE));
}
} finally {
tbl.readUnlock();
}
- execSQLs(partitionAnalysisSQLs, params);
+ execSQLs(partitionAnalysisSQLs);
+ params.remove("partId");
+ params.put("type", col.getType().toString());
+ StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
+ String sql = stringSubstitutor.replace(ANALYZE_COLUMN_SQL_TEMPLATE);
+ execSQL(sql);
+ }
+
+ @VisibleForTesting
+ public void execSQLs(List<String> partitionAnalysisSQLs) throws Exception {
+ for (String sql : partitionAnalysisSQLs) {
+ execSQL(sql);
+ }
}
@VisibleForTesting
- public void execSQLs(List<String> partitionAnalysisSQLs, Map<String, String> params) throws Exception {
+ public void execSQL(String sql) throws Exception {
+ if (killed) {
+ return;
+ }
long startTime = System.currentTimeMillis();
- LOG.debug("analyze task {} start at {}", info.toString(), new Date());
- try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext(info.jobType.equals(JobType.SYSTEM))) {
- List<List<String>> sqlGroups = Lists.partition(partitionAnalysisSQLs, StatisticConstants.UNION_ALL_LIMIT);
- for (List<String> group : sqlGroups) {
- if (killed) {
- return;
- }
- StringJoiner partitionCollectSQL = new StringJoiner("UNION ALL");
- group.forEach(partitionCollectSQL::add);
- stmtExecutor = new StmtExecutor(r.connectContext, partitionCollectSQL.toString());
- buf.add(stmtExecutor.executeInternalQuery()
- .stream().map(ColStatsData::new).collect(Collectors.toList()));
- QueryState queryState = r.connectContext.getState();
- if (queryState.getStateType().equals(MysqlStateType.ERR)) {
- throw new RuntimeException(String.format("Failed to analyze %s.%s.%s, error: %s sql: %s",
- info.catalogName, info.dbName, info.colName, partitionCollectSQL,
- queryState.getErrorMessage()));
- }
- }
- for (List<ColStatsData> colStatsDataList : buf) {
- StringBuilder batchInsertSQL =
- new StringBuilder("INSERT INTO " + StatisticConstants.FULL_QUALIFIED_STATS_TBL_NAME
- + " VALUES ");
- StringJoiner sj = new StringJoiner(",");
- colStatsDataList.forEach(c -> sj.add(c.toSQL(true)));
- batchInsertSQL.append(sj.toString());
- stmtExecutor = new StmtExecutor(r.connectContext, batchInsertSQL.toString());
- executeWithExceptionOnFail(stmtExecutor);
- }
- params.put("type", col.getType().toString());
- StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
- String sql = stringSubstitutor.replace(ANALYZE_COLUMN_SQL_TEMPLATE);
+ LOG.info("ANALYZE SQL : " + sql + " start at " + startTime);
+ try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext()) {
+ r.connectContext.getSessionVariable().disableNereidsPlannerOnce();
stmtExecutor = new StmtExecutor(r.connectContext, sql);
- executeWithExceptionOnFail(stmtExecutor);
+ r.connectContext.setExecutor(stmtExecutor);
+ stmtExecutor.execute();
+ QueryState queryState = r.connectContext.getState();
+ if (queryState.getStateType().equals(MysqlStateType.ERR)) {
+ throw new RuntimeException(String.format("Failed to analyze %s.%s.%s, error: %s sql: %s",
+ info.catalogName, info.dbName, info.colName, sql, queryState.getErrorMessage()));
+ }
} finally {
- LOG.debug("analyze task {} end. cost {}ms", info,
- System.currentTimeMillis() - startTime);
+ LOG.info("Analyze SQL: " + sql + " cost time: " + (System.currentTimeMillis() - startTime) + "ms");
}
-
}
+
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ResultRow.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ResultRow.java
deleted file mode 100644
index 9945175a228..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ResultRow.java
+++ /dev/null
@@ -1,59 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.statistics;
-
-import com.google.gson.annotations.SerializedName;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.StringJoiner;
-
-public class ResultRow {
- @SerializedName("values")
- private final List<String> values;
-
- public ResultRow(List<String> values) {
- this.values = values;
- }
-
- public List<String> getValues() {
- return values != null ? values : Collections.emptyList();
- }
-
- @Override
- public String toString() {
- StringJoiner sj = new StringJoiner(",", "ResultRow:{", "}");
- for (String val : values) {
- sj.add(val);
- }
- return sj.toString();
- }
-
- public String get(int idx) {
- return values.get(idx);
- }
-
- /**
- * If analyze an empty table, some stats would be null, return a default value
- * to avoid npe would deserialize it.
- */
- public String getWithDefault(int idx, String defaultVal) {
- String val = values.get(idx);
- return val == null ? defaultVal : val;
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
index e6b8297d0c0..a2194834030 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
@@ -17,9 +17,6 @@
package org.apache.doris.statistics;
-import org.apache.doris.catalog.Column;
-import org.apache.doris.catalog.OlapTable;
-import org.apache.doris.catalog.TableIf;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.FeConstants;
import org.apache.doris.system.SystemInfoService;
@@ -29,16 +26,22 @@ import java.util.List;
import java.util.concurrent.TimeUnit;
public class StatisticConstants {
+ public static final String ANALYSIS_TBL_NAME = "table_statistics";
public static final String STATISTIC_TBL_NAME = "column_statistics";
+
public static final String HISTOGRAM_TBL_NAME = "histogram_statistics";
public static final int MAX_NAME_LEN = 64;
public static final int ID_LEN = 4096;
+ public static final int STATISTICS_CACHE_VALID_DURATION_IN_HOURS = 24 * 2;
+
public static final int STATISTICS_CACHE_REFRESH_INTERVAL = 24 * 2;
+ public static final int ROW_COUNT_CACHE_VALID_DURATION_IN_HOURS = 12;
+
/**
* Bucket count fot column_statistics and analysis_job table.
*/
@@ -60,51 +63,26 @@ public class StatisticConstants {
public static final int HISTOGRAM_MAX_BUCKET_NUM = 128;
- public static final int ANALYZE_MANAGER_INTERVAL_IN_SECS = 60;
+ /**
+ * The health of the table indicates the health of the table statistics, rang in [0, 100].
+ * Below this threshold will automatically re-collect statistics. TODO make it in fe.conf
+ */
+ public static final int TABLE_STATS_HEALTH_THRESHOLD = 80;
- public static List<String> SYSTEM_DBS = new ArrayList<>();
+ public static final int ANALYZE_MANAGER_INTERVAL_IN_SECS = 60;
- public static int ANALYZE_TASK_RETRY_TIMES = 5;
+ public static List<String> STATISTICS_DB_BLACK_LIST = new ArrayList<>();
public static final String DB_NAME = SystemInfoService.DEFAULT_CLUSTER + ":" + FeConstants.INTERNAL_DB_NAME;
- public static final String FULL_QUALIFIED_STATS_TBL_NAME = FeConstants.INTERNAL_DB_NAME + "." + STATISTIC_TBL_NAME;
-
public static final int STATISTIC_INTERNAL_TABLE_REPLICA_NUM = 3;
- public static final int RETRY_LOAD_QUEUE_SIZE = 1000;
-
- public static final int RETRY_LOAD_THREAD_POOL_SIZE = 1;
-
- public static final int LOAD_RETRY_TIMES = 3;
-
- // union more relation than 512 may cause StackOverFlowException in the future.
- public static final int UNION_ALL_LIMIT = 512;
-
- public static final String FULL_AUTO_ANALYZE_START_TIME = "00:00:00";
- public static final String FULL_AUTO_ANALYZE_END_TIME = "23:59:59";
+ public static int ANALYZE_TASK_RETRY_TIMES = 3;
static {
- SYSTEM_DBS.add(SystemInfoService.DEFAULT_CLUSTER
+ STATISTICS_DB_BLACK_LIST.add(SystemInfoService.DEFAULT_CLUSTER
+ ClusterNamespace.CLUSTER_DELIMITER + FeConstants.INTERNAL_DB_NAME);
- SYSTEM_DBS.add(SystemInfoService.DEFAULT_CLUSTER
+ STATISTICS_DB_BLACK_LIST.add(SystemInfoService.DEFAULT_CLUSTER
+ ClusterNamespace.CLUSTER_DELIMITER + "information_schema");
}
-
- public static boolean isSystemTable(TableIf tableIf) {
- if (tableIf instanceof OlapTable) {
- OlapTable olapTable = (OlapTable) tableIf;
- if (StatisticConstants.SYSTEM_DBS.contains(olapTable.getQualifiedDbName())) {
- return true;
- }
- }
- return false;
- }
-
- public static boolean shouldIgnoreCol(TableIf tableIf, Column c) {
- if (isSystemTable(tableIf)) {
- return true;
- }
- return !c.isVisible();
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticRange.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticRange.java
index 74b77c2ee7c..d3bdbdabd0b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticRange.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticRange.java
@@ -17,8 +17,6 @@
package org.apache.doris.statistics;
-import org.apache.doris.analysis.LiteralExpr;
-import org.apache.doris.common.Pair;
import org.apache.doris.nereids.types.DataType;
import java.util.Objects;
@@ -31,47 +29,20 @@ public class StatisticRange {
* {@code NaN} represents empty range ({@code high} must be {@code NaN} too)
*/
private final double low;
-
- private final LiteralExpr lowExpr;
/**
* {@code NaN} represents empty range ({@code low} must be {@code NaN} too)
*/
private final double high;
- private final LiteralExpr highExpr;
-
private final double distinctValues;
private final DataType dataType;
- private final boolean isEmpty;
-
- public StatisticRange(double low, LiteralExpr lowExpr, double high, LiteralExpr highExpr,
- double distinctValues, DataType dataType) {
- this(low, lowExpr, high, highExpr, distinctValues, dataType, false);
- }
-
- private StatisticRange(double low, LiteralExpr lowExpr, double high, LiteralExpr highExpr,
- double distinctValues, DataType dataType, boolean isEmpty) {
+ public StatisticRange(double low, double high, double distinctValues, DataType dataType) {
this.low = low;
- this.lowExpr = lowExpr;
this.high = high;
- this.highExpr = highExpr;
this.distinctValues = distinctValues;
this.dataType = dataType;
- this.isEmpty = isEmpty;
- }
-
- public LiteralExpr getLowExpr() {
- return lowExpr;
- }
-
- public LiteralExpr getHighExpr() {
- return highExpr;
- }
-
- public DataType getDataType() {
- return dataType;
}
public double overlapPercentWith(StatisticRange other) {
@@ -108,29 +79,19 @@ public class StatisticRange {
}
public static StatisticRange empty(DataType dataType) {
- return new StatisticRange(Double.NEGATIVE_INFINITY, null, Double.POSITIVE_INFINITY,
- null, 0, dataType, true);
+ return new StatisticRange(Double.NaN, Double.NaN, 0, dataType);
}
public boolean isEmpty() {
- return isEmpty;
+ return Double.isNaN(low) && Double.isNaN(high);
}
public boolean isBothInfinite() {
return Double.isInfinite(low) && Double.isInfinite(high);
}
- public boolean isInfinite() {
- return Double.isInfinite(low) || Double.isInfinite(high);
- }
-
- public boolean isFinite() {
- return Double.isFinite(low) && Double.isFinite(high);
- }
-
- public static StatisticRange from(ColumnStatistic colStats, DataType dataType) {
- return new StatisticRange(colStats.minValue, colStats.minExpr, colStats.maxValue, colStats.maxExpr,
- colStats.ndv, dataType);
+ public static StatisticRange from(ColumnStatistic column, DataType dataType) {
+ return new StatisticRange(column.minValue, column.maxValue, column.ndv, dataType);
}
public double getLow() {
@@ -146,49 +107,22 @@ public class StatisticRange {
}
public StatisticRange intersect(StatisticRange other) {
- Pair<Double, LiteralExpr> biggerLow = maxPair(low, lowExpr, other.low, other.lowExpr);
- double newLow = biggerLow.first;
- LiteralExpr newLowExpr = biggerLow.second;
-
- Pair<Double, LiteralExpr> smallerHigh = minPair(high, highExpr, other.high, other.highExpr);
- double newHigh = smallerHigh.first;
- LiteralExpr newHighExpr = smallerHigh.second;
+ double newLow = Math.max(low, other.low);
+ double newHigh = Math.min(high, other.high);
if (newLow <= newHigh) {
- return new StatisticRange(newLow, newLowExpr, newHigh, newHighExpr,
- overlappingDistinctValues(other), dataType);
+ return new StatisticRange(newLow, newHigh, overlappingDistinctValues(other), dataType);
}
return empty(dataType);
}
- public Pair<Double, LiteralExpr> minPair(double r1, LiteralExpr e1, double r2, LiteralExpr e2) {
- if (r1 < r2) {
- return Pair.of(r1, e1);
- }
- return Pair.of(r2, e2);
- }
-
- public Pair<Double, LiteralExpr> maxPair(double r1, LiteralExpr e1, double r2, LiteralExpr e2) {
- if (r1 > r2) {
- return Pair.of(r1, e1);
- }
- return Pair.of(r2, e2);
- }
-
public StatisticRange cover(StatisticRange other) {
- // double newLow = Math.max(low, other.low);
- // double newHigh = Math.min(high, other.high);
- Pair<Double, LiteralExpr> biggerLow = maxPair(low, lowExpr, other.low, other.lowExpr);
- double newLow = biggerLow.first;
- LiteralExpr newLowExpr = biggerLow.second;
- Pair<Double, LiteralExpr> smallerHigh = minPair(high, highExpr, other.high, other.highExpr);
- double newHigh = smallerHigh.first;
- LiteralExpr newHighExpr = smallerHigh.second;
-
+ double newLow = Math.max(low, other.low);
+ double newHigh = Math.min(high, other.high);
if (newLow <= newHigh) {
double overlapPercentOfLeft = overlapPercentWith(other);
double overlapDistinctValuesLeft = overlapPercentOfLeft * distinctValues;
double coveredDistinctValues = minExcludeNaN(distinctValues, overlapDistinctValuesLeft);
- return new StatisticRange(newLow, newLowExpr, newHigh, newHighExpr, coveredDistinctValues, dataType);
+ return new StatisticRange(newLow, newHigh, coveredDistinctValues, dataType);
}
return empty(dataType);
}
@@ -201,10 +135,7 @@ public class StatisticRange {
double maxOverlapNDV = Math.max(overlapNDVThis, overlapNDVOther);
double newNDV = maxOverlapNDV + ((1 - overlapPercentThis) * distinctValues)
+ ((1 - overlapPercentOther) * other.distinctValues);
- Pair<Double, LiteralExpr> smallerMin = minPair(low, lowExpr, other.low, other.lowExpr);
- Pair<Double, LiteralExpr> biggerHigh = maxPair(high, highExpr, other.high, other.highExpr);
- return new StatisticRange(smallerMin.first, smallerMin.second,
- biggerHigh.first, biggerHigh.second, newNDV, dataType);
+ return new StatisticRange(Math.min(low, other.low), Math.max(high, other.high), newNDV, dataType);
}
private double overlappingDistinctValues(StatisticRange other) {
@@ -239,4 +170,7 @@ public class StatisticRange {
return distinctValues;
}
+ public static StatisticRange fromColumnStatistics(ColumnStatistic columnStatistic, DataType dataType) {
+ return new StatisticRange(columnStatistic.minValue, columnStatistic.maxValue, columnStatistic.ndv, dataType);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticalType.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticalType.java
index 7fe9b03cbcf..67dd9bb0543 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticalType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticalType.java
@@ -22,7 +22,6 @@ public enum StatisticalType {
AGG_NODE,
ANALYTIC_EVAL_NODE,
ASSERT_NUM_ROWS_NODE,
- CTE_SCAN_NODE,
BROKER_SCAN_NODE,
NESTED_LOOP_JOIN_NODE,
EMPTY_SET_NODE,
@@ -55,5 +54,4 @@ public enum StatisticalType {
METADATA_SCAN_NODE,
JDBC_SCAN_NODE,
TEST_EXTERNAL_TABLE,
- GROUP_COMMIT_SCAN_NODE
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java
index 77c221f5931..5c628aaba30 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java
@@ -17,18 +17,17 @@
package org.apache.doris.statistics;
+import org.apache.doris.nereids.stats.ExpressionEstimation;
import org.apache.doris.nereids.stats.StatsMathUtil;
import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.Slot;
import java.text.DecimalFormat;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Set;
public class Statistics {
- private static final int K_BYTES = 1024;
+ private static int K_BYTES = 1024;
private final double rowCount;
@@ -37,10 +36,33 @@ public class Statistics {
// the byte size of one tuple
private double tupleSize;
+ @Deprecated
+ private double width;
+
+ @Deprecated
+ private double penalty;
+
+ /**
+ * after filter, compute the new ndv of a column
+ * @param ndv original ndv of column
+ * @param newRowCount the row count of table after filter
+ * @param oldRowCount the row count of table before filter
+ * @return the new ndv after filter
+ */
+ public static double computeNdv(double ndv, double newRowCount, double oldRowCount) {
+ if (newRowCount > oldRowCount) {
+ return ndv;
+ }
+ double selectOneTuple = newRowCount / StatsMathUtil.nonZeroDivisor(oldRowCount);
+ double allTuplesOfSameDistinctValueNotSelected = Math.pow((1 - selectOneTuple), oldRowCount / ndv);
+ return Math.min(ndv * (1 - allTuplesOfSameDistinctValueNotSelected), newRowCount);
+ }
+
public Statistics(Statistics another) {
this.rowCount = another.rowCount;
this.expressionToColumnStats = new HashMap<>(another.expressionToColumnStats);
- this.tupleSize = another.tupleSize;
+ this.width = another.width;
+ this.penalty = another.penalty;
}
public Statistics(double rowCount, Map<Expression, ColumnStatistic> expressionToColumnStats) {
@@ -48,6 +70,14 @@ public class Statistics {
this.expressionToColumnStats = expressionToColumnStats;
}
+ public Statistics(double rowCount, Map<Expression, ColumnStatistic> expressionToColumnStats, double width,
+ double penalty) {
+ this.rowCount = rowCount;
+ this.expressionToColumnStats = expressionToColumnStats;
+ this.width = width;
+ this.penalty = penalty;
+ }
+
public ColumnStatistic findColumnStatistics(Expression expression) {
return expressionToColumnStats.get(expression);
}
@@ -60,46 +90,53 @@ public class Statistics {
return rowCount;
}
+ /*
+ * Return a stats with new rowCount and fix each column stats.
+ */
public Statistics withRowCount(double rowCount) {
- return new Statistics(rowCount, new HashMap<>(expressionToColumnStats));
+ if (Double.isNaN(rowCount)) {
+ return this;
+ }
+ Statistics statistics = new Statistics(rowCount, new HashMap<>(expressionToColumnStats), width, penalty);
+ statistics.fix(rowCount, StatsMathUtil.nonZeroDivisor(this.rowCount));
+ return statistics;
}
/**
* Update by count.
*/
- public Statistics withRowCountAndEnforceValid(double rowCount) {
+ public Statistics updateRowCountOnly(double rowCount) {
Statistics statistics = new Statistics(rowCount, expressionToColumnStats);
- statistics.enforceValid();
+ for (Entry<Expression, ColumnStatistic> entry : expressionToColumnStats.entrySet()) {
+ ColumnStatistic columnStatistic = entry.getValue();
+ ColumnStatisticBuilder columnStatisticBuilder = new ColumnStatisticBuilder(columnStatistic);
+ columnStatisticBuilder.setNdv(Math.min(columnStatistic.ndv, rowCount));
+ double nullFactor = (rowCount - columnStatistic.numNulls) / rowCount;
+ columnStatisticBuilder.setNumNulls(nullFactor * rowCount);
+ columnStatisticBuilder.setCount(rowCount);
+ statistics.addColumnStats(entry.getKey(), columnStatisticBuilder.build());
+ }
return statistics;
}
- public void enforceValid() {
+ /**
+ * Fix by sel.
+ */
+ public void fix(double newRowCount, double originRowCount) {
+ double sel = newRowCount / originRowCount;
for (Entry<Expression, ColumnStatistic> entry : expressionToColumnStats.entrySet()) {
ColumnStatistic columnStatistic = entry.getValue();
- if (!checkColumnStatsValid(columnStatistic)) {
- double ndv = Math.min(columnStatistic.ndv, rowCount);
- ColumnStatisticBuilder columnStatisticBuilder = new ColumnStatisticBuilder(columnStatistic);
- columnStatisticBuilder.setNdv(ndv);
- columnStatisticBuilder.setNumNulls(Math.min(columnStatistic.numNulls, rowCount - ndv));
- columnStatisticBuilder.setCount(rowCount);
- columnStatistic = columnStatisticBuilder.build();
- expressionToColumnStats.put(entry.getKey(), columnStatistic);
- }
+ ColumnStatisticBuilder columnStatisticBuilder = new ColumnStatisticBuilder(columnStatistic);
+ columnStatisticBuilder.setNdv(computeNdv(columnStatistic.ndv, newRowCount, originRowCount));
+ columnStatisticBuilder.setNumNulls(Math.min(columnStatistic.numNulls * sel, newRowCount));
+ columnStatisticBuilder.setCount(newRowCount);
+ expressionToColumnStats.put(entry.getKey(), columnStatisticBuilder.build());
}
}
- public boolean checkColumnStatsValid(ColumnStatistic columnStatistic) {
- return columnStatistic.ndv <= rowCount
- && columnStatistic.numNulls <= rowCount - columnStatistic.ndv;
- }
-
public Statistics withSel(double sel) {
sel = StatsMathUtil.minNonNaN(sel, 1);
- if (Double.isNaN(rowCount)) {
- return this;
- }
- double newCount = rowCount * sel;
- return new Statistics(newCount, new HashMap<>(expressionToColumnStats));
+ return withRowCount(rowCount * sel);
}
public Statistics addColumnStats(Expression expression, ColumnStatistic columnStatistic) {
@@ -107,10 +144,9 @@ public class Statistics {
return this;
}
- public boolean isInputSlotsUnknown(Set<Slot> inputs) {
- return inputs.stream()
- .allMatch(s -> expressionToColumnStats.containsKey(s)
- && expressionToColumnStats.get(s).isUnKnown);
+ public Statistics merge(Statistics statistics) {
+ expressionToColumnStats.putAll(statistics.expressionToColumnStats);
+ return this;
}
private double computeTupleSize() {
@@ -147,20 +183,53 @@ public class Statistics {
return format.format(rowCount);
}
+ public void setWidth(double width) {
+ this.width = width;
+ }
+
+ public void setPenalty(double penalty) {
+ this.penalty = penalty;
+ }
+
+ public double getWidth() {
+ return width;
+ }
+
+ public double getPenalty() {
+ return penalty;
+ }
+
public int getBENumber() {
return 1;
}
public static Statistics zero(Statistics statistics) {
Statistics zero = new Statistics(0, new HashMap<>());
- for (Entry<Expression, ColumnStatistic> entry : statistics.expressionToColumnStats.entrySet()) {
+ for (Map.Entry<Expression, ColumnStatistic> entry : statistics.expressionToColumnStats.entrySet()) {
zero.addColumnStats(entry.getKey(), ColumnStatistic.ZERO);
}
return zero;
}
+ public boolean almostUniqueExpression(Expression expr) {
+ ExpressionEstimation estimator = new ExpressionEstimation();
+ double ndvErrorThreshold = 0.9;
+ ColumnStatistic colStats = expr.accept(estimator, this);
+ if (colStats.ndv > colStats.count * ndvErrorThreshold) {
+ return true;
+ }
+ return false;
+ }
+
+ public boolean isStatsUnknown(Expression expr) {
+ ExpressionEstimation estimator = new ExpressionEstimation();
+ ColumnStatistic colStats = expr.accept(estimator, this);
+ return colStats.isUnKnown;
+ }
+
/**
* merge this and other colStats.ndv, choose min
+ * @param other
*/
public void updateNdv(Statistics other) {
for (Expression expr : expressionToColumnStats.keySet()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java
new file mode 100644
index 00000000000..aae783ca8b2
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java
@@ -0,0 +1,215 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.statistics;
+
+import org.apache.doris.analysis.DdlStmt;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.Partition;
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.common.Config;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.util.MasterDaemon;
+import org.apache.doris.statistics.AnalysisInfo.JobType;
+import org.apache.doris.statistics.util.StatisticsUtil;
+
+import com.google.common.collect.Maps;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+public class StatisticsAutoAnalyzer extends MasterDaemon {
+
+ private static final Logger LOG = LogManager.getLogger(StatisticsAutoAnalyzer.class);
+
+ public StatisticsAutoAnalyzer() {
+ super("Automatic Analyzer", TimeUnit.MINUTES.toMillis(Config.auto_check_statistics_in_minutes));
+ }
+
+ @Override
+ protected void runAfterCatalogReady() {
+ if (!Env.getCurrentEnv().isMaster()) {
+ return;
+ }
+ if (!StatisticsUtil.statsTblAvailable()) {
+ return;
+ }
+ if (Config.enable_auto_collect_statistics) {
+ analyzePeriodically();
+ analyzeAutomatically();
+ }
+ }
+
+ public void autoAnalyzeStats(DdlStmt ddlStmt) {
+ // TODO Monitor some DDL statements, and then trigger automatic analysis tasks
+ }
+
+ private void analyzePeriodically() {
+ try {
+ AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
+ List<AnalysisInfo> jobInfos = analysisManager.findPeriodicJobs();
+ for (AnalysisInfo jobInfo : jobInfos) {
+ jobInfo = new AnalysisInfoBuilder(jobInfo).setJobType(JobType.SYSTEM).build();
+ analysisManager.createAnalysisJob(jobInfo);
+ }
+ } catch (DdlException e) {
+ LOG.warn("Failed to periodically analyze the statistics." + e);
+ }
+ }
+
+ private void analyzeAutomatically() {
+ AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
+ List<AnalysisInfo> jobInfos = analysisManager.findAutomaticAnalysisJobs();
+ for (AnalysisInfo jobInfo : jobInfos) {
+ AnalysisInfo checkedJobInfo = null;
+ try {
+ checkedJobInfo = checkAutomaticJobInfo(jobInfo);
+ if (checkedJobInfo != null) {
+ analysisManager.createAnalysisJob(checkedJobInfo);
+ }
+ } catch (Throwable t) {
+ LOG.warn("Failed to create analyze job: {}", checkedJobInfo);
+ }
+
+ }
+ }
+
+ /**
+ * Check if automatic analysis of statistics is required.
+ * <p>
+ * Step1: check the health of the table, if the health is good,
+ * there is no need to re-analyze, or check partition
+ * <p>
+ * Step2: check the partition update time, if the partition is not updated
+ * after the statistics is analyzed, there is no need to re-analyze
+ * <p>
+ * Step3: if the partition is updated after the statistics is analyzed,
+ * check the health of the partition, if the health is good, there is no need to re-analyze
+ * - Step3.1: check the analyzed partition statistics
+ * - Step3.2: Check for new partitions for which statistics were not analyzed
+ * <p>
+ * TODO new columns is not currently supported to analyze automatically
+ *
+ * @param jobInfo analysis job info
+ * @return new job info after check
+ * @throws Throwable failed to check
+ */
+ private AnalysisInfo checkAutomaticJobInfo(AnalysisInfo jobInfo) throws Throwable {
+ long lastExecTimeInMs = jobInfo.lastExecTimeInMs;
+ TableIf table = StatisticsUtil
+ .findTable(jobInfo.catalogName, jobInfo.dbName, jobInfo.tblName);
+ TableStatistic tblStats = StatisticsRepository.fetchTableLevelStats(table.getId());
+
+ if (tblStats == TableStatistic.UNKNOWN) {
+ LOG.warn("Failed to automatically analyze statistics, "
+ + "no corresponding table statistics for job: {}", jobInfo.toString());
+ throw new DdlException("No corresponding table statistics for automatic job.");
+ }
+
+ if (!needReanalyzeTable(table, tblStats)) {
+ return null;
+ }
+
+ Set<String> needRunPartitions = new HashSet<>();
+ Set<String> statsPartitions = jobInfo.colToPartitions.values()
+ .stream()
+ .flatMap(Collection::stream)
+ .collect(Collectors.toSet());
+
+ checkAnalyzedPartitions(table, statsPartitions, needRunPartitions, lastExecTimeInMs);
+ checkNewPartitions(table, needRunPartitions, lastExecTimeInMs);
+
+ if (needRunPartitions.isEmpty()) {
+ return null;
+ }
+
+ return getAnalysisJobInfo(jobInfo, table, needRunPartitions);
+ }
+
+ private boolean needReanalyzeTable(TableIf table, TableStatistic tblStats) {
+ long rowCount = table.getRowCount();
+ long updateRows = Math.abs(rowCount - tblStats.rowCount);
+ int tblHealth = StatisticsUtil.getTableHealth(rowCount, updateRows);
+ return tblHealth < StatisticConstants.TABLE_STATS_HEALTH_THRESHOLD;
+ }
+
+ private void checkAnalyzedPartitions(TableIf table, Set<String> statsPartitions,
+ Set<String> needRunPartitions, long lastExecTimeInMs) throws DdlException {
+ for (String statsPartition : statsPartitions) {
+ Partition partition = table.getPartition(statsPartition);
+ if (partition == null) {
+ // Partition that has been deleted also need to
+ // be reanalyzed (delete partition statistics later)
+ needRunPartitions.add(statsPartition);
+ continue;
+ }
+ TableStatistic partitionStats = StatisticsRepository
+ .fetchTableLevelOfPartStats(partition.getId());
+ if (partitionStats == TableStatistic.UNKNOWN) {
+ continue;
+ }
+ if (needReanalyzePartition(lastExecTimeInMs, partition, partitionStats)) {
+ needRunPartitions.add(partition.getName());
+ }
+ }
+ }
+
+ private boolean needReanalyzePartition(long lastExecTimeInMs, Partition partition, TableStatistic partStats) {
+ long partUpdateTime = partition.getVisibleVersionTime();
+ if (partUpdateTime < lastExecTimeInMs) {
+ return false;
+ }
+ long pRowCount = partition.getBaseIndex().getRowCount();
+ long pUpdateRows = Math.abs(pRowCount - partStats.rowCount);
+ int partHealth = StatisticsUtil.getTableHealth(pRowCount, pUpdateRows);
+ return partHealth < StatisticConstants.TABLE_STATS_HEALTH_THRESHOLD;
+ }
+
+ private void checkNewPartitions(TableIf table, Set<String> needRunPartitions, long lastExecTimeInMs) {
+ Set<String> partitionNames = table.getPartitionNames();
+ partitionNames.removeAll(needRunPartitions);
+ needRunPartitions.addAll(
+ partitionNames.stream()
+ .map(table::getPartition)
+ .filter(partition -> partition.getVisibleVersionTime() >= lastExecTimeInMs)
+ .map(Partition::getName)
+ .collect(Collectors.toSet())
+ );
+ }
+
+ private AnalysisInfo getAnalysisJobInfo(AnalysisInfo jobInfo, TableIf table,
+ Set<String> needRunPartitions) {
+ Map<String, Set<String>> newColToPartitions = Maps.newHashMap();
+ Map<String, Set<String>> colToPartitions = jobInfo.colToPartitions;
+ colToPartitions.keySet().forEach(colName -> {
+ Column column = table.getColumn(colName);
+ if (column != null) {
+ newColToPartitions.put(colName, needRunPartitions);
+ }
+ });
+ return new AnalysisInfoBuilder(jobInfo)
+ .setColToPartitions(newColToPartitions).build();
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
deleted file mode 100644
index fe535b0fb4a..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++ /dev/null
@@ -1,196 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.statistics;
-
-import org.apache.doris.analysis.TableName;
-import org.apache.doris.catalog.Column;
-import org.apache.doris.catalog.DatabaseIf;
-import org.apache.doris.catalog.Env;
-import org.apache.doris.catalog.OlapTable;
-import org.apache.doris.catalog.TableIf;
-import org.apache.doris.catalog.external.ExternalTable;
-import org.apache.doris.common.Config;
-import org.apache.doris.common.util.TimeUtils;
-import org.apache.doris.datasource.CatalogIf;
-import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod;
-import org.apache.doris.statistics.AnalysisInfo.JobType;
-import org.apache.doris.statistics.AnalysisInfo.ScheduleType;
-import org.apache.doris.statistics.util.StatisticsUtil;
-
-import com.google.common.collect.Maps;
-import org.apache.hudi.common.util.VisibleForTesting;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.time.LocalTime;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-public class StatisticsAutoCollector extends StatisticsCollector {
-
- private static final Logger LOG = LogManager.getLogger(StatisticsAutoCollector.class);
-
- public StatisticsAutoCollector() {
- super("Automatic Analyzer",
- TimeUnit.MINUTES.toMillis(Config.auto_check_statistics_in_minutes),
- new AnalysisTaskExecutor(Config.full_auto_analyze_simultaneously_running_task_num));
- }
-
- @Override
- protected void collect() {
- if (!StatisticsUtil.inAnalyzeTime(LocalTime.now(TimeUtils.getTimeZone().toZoneId()))) {
- analysisTaskExecutor.clear();
- return;
- }
- if (StatisticsUtil.enableAutoAnalyze()) {
- analyzeAll();
- }
- }
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- private void analyzeAll() {
- Set<CatalogIf> catalogs = Env.getCurrentEnv().getCatalogMgr().getCopyOfCatalog();
- for (CatalogIf ctl : catalogs) {
- if (!ctl.enableAutoAnalyze()) {
- continue;
- }
- Collection<DatabaseIf> dbs = ctl.getAllDbs();
- for (DatabaseIf<TableIf> databaseIf : dbs) {
- if (StatisticConstants.SYSTEM_DBS.contains(databaseIf.getFullName())) {
- continue;
- }
- analyzeDb(databaseIf);
- }
- }
- }
-
- public void analyzeDb(DatabaseIf<TableIf> databaseIf) {
- List<AnalysisInfo> analysisInfos = constructAnalysisInfo(databaseIf);
- for (AnalysisInfo analysisInfo : analysisInfos) {
- analysisInfo = getReAnalyzeRequiredPart(analysisInfo);
- if (analysisInfo == null) {
- continue;
- }
- try {
- createSystemAnalysisJob(analysisInfo);
- } catch (Exception e) {
- LOG.warn("Failed to create analysis job", e);
- }
- }
- }
-
- protected List<AnalysisInfo> constructAnalysisInfo(DatabaseIf<? extends TableIf> db) {
- List<AnalysisInfo> analysisInfos = new ArrayList<>();
- for (TableIf table : db.getTables()) {
- if (skip(table)) {
- continue;
- }
- createAnalyzeJobForTbl(db, analysisInfos, table);
- }
- return analysisInfos;
- }
-
- // return true if skip auto analyze this time.
- protected boolean skip(TableIf table) {
- if (!(table instanceof OlapTable || table instanceof ExternalTable)) {
- return true;
- }
- if (table.getDataSize(true) < Config.huge_table_lower_bound_size_in_bytes) {
- return false;
- }
- TableStatsMeta tableStats = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(table.getId());
- return System.currentTimeMillis() - tableStats.updatedTime < Config.huge_table_auto_analyze_interval_in_millis;
- }
-
- protected void createAnalyzeJobForTbl(DatabaseIf<? extends TableIf> db,
- List<AnalysisInfo> analysisInfos, TableIf table) {
- AnalysisMethod analysisMethod = table.getDataSize(true) > Config.huge_table_lower_bound_size_in_bytes
- ? AnalysisMethod.SAMPLE : AnalysisMethod.FULL;
- TableName tableName = new TableName(db.getCatalog().getName(), db.getFullName(),
- table.getName());
- AnalysisInfo jobInfo = new AnalysisInfoBuilder()
- .setJobId(Env.getCurrentEnv().getNextId())
- .setCatalogName(db.getCatalog().getName())
- .setDbName(db.getFullName())
- .setTblName(tableName.getTbl())
- .setColName(
- table.getBaseSchema().stream().filter(c -> !StatisticsUtil.isUnsupportedType(c.getType()))
- .map(
- Column::getName).collect(Collectors.joining(","))
- )
- .setAnalysisType(AnalysisInfo.AnalysisType.FUNDAMENTALS)
- .setAnalysisMode(AnalysisInfo.AnalysisMode.INCREMENTAL)
- .setAnalysisMethod(analysisMethod)
- .setSampleRows(Config.huge_table_default_sample_rows)
- .setScheduleType(ScheduleType.AUTOMATIC)
- .setState(AnalysisState.PENDING)
- .setTaskIds(new ArrayList<>())
- .setLastExecTimeInMs(System.currentTimeMillis())
- .setJobType(JobType.SYSTEM).build();
- analysisInfos.add(jobInfo);
- }
-
- @VisibleForTesting
- protected AnalysisInfo getReAnalyzeRequiredPart(AnalysisInfo jobInfo) {
- TableIf table = StatisticsUtil
- .findTable(jobInfo.catalogName, jobInfo.dbName, jobInfo.tblName);
- AnalysisManager analysisManager = Env.getServingEnv().getAnalysisManager();
- TableStatsMeta tblStats = analysisManager.findTableStatsStatus(table.getId());
-
- if (!table.needReAnalyzeTable(tblStats)) {
- return null;
- }
-
- Map<String, Set<String>> needRunPartitions = table.findReAnalyzeNeededPartitions();
-
- if (needRunPartitions.isEmpty()) {
- return null;
- }
-
- return new AnalysisInfoBuilder(jobInfo).setColToPartitions(needRunPartitions).build();
- }
-
- @VisibleForTesting
- protected AnalysisInfo getAnalysisJobInfo(AnalysisInfo jobInfo, TableIf table,
- Set<String> needRunPartitions) {
- Map<String, Set<String>> newColToPartitions = Maps.newHashMap();
- Map<String, Set<String>> colToPartitions = jobInfo.colToPartitions;
- if (colToPartitions == null) {
- for (Column c : table.getColumns()) {
- if (StatisticsUtil.isUnsupportedType(c.getType())) {
- continue;
- }
- newColToPartitions.put(c.getName(), needRunPartitions);
- }
- } else {
- colToPartitions.keySet().forEach(colName -> {
- Column column = table.getColumn(colName);
- if (column != null) {
- newColToPartitions.put(colName, needRunPartitions);
- }
- });
- }
- return new AnalysisInfoBuilder(jobInfo)
- .setColToPartitions(newColToPartitions).build();
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsBuilder.java
index a0e75f7df38..3622006542d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsBuilder.java
@@ -26,7 +26,7 @@ public class StatisticsBuilder {
private double rowCount;
- private final Map<Expression, ColumnStatistic> expressionToColumnStats;
+ private Map<Expression, ColumnStatistic> expressionToColumnStats;
public StatisticsBuilder() {
expressionToColumnStats = new HashMap<>();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
index c9b049a8cfc..cb9e4ca3228 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
@@ -24,6 +24,7 @@ import org.apache.doris.common.ThreadPoolManager;
import org.apache.doris.ha.FrontendNodeType;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.statistics.util.InternalQueryResult.ResultRow;
import org.apache.doris.statistics.util.StatisticsUtil;
import org.apache.doris.system.Frontend;
import org.apache.doris.thrift.FrontendService;
@@ -33,21 +34,16 @@ import org.apache.doris.thrift.TUpdateFollowerStatsCacheRequest;
import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
import com.github.benmanes.caffeine.cache.Caffeine;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.hudi.common.util.VisibleForTesting;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.time.Duration;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
-import java.util.Map;
import java.util.Optional;
-import java.util.Set;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
public class StatisticsCache {
@@ -63,6 +59,7 @@ public class StatisticsCache {
private final ColumnStatisticsCacheLoader columnStatisticsCacheLoader = new ColumnStatisticsCacheLoader();
private final HistogramCacheLoader histogramCacheLoader = new HistogramCacheLoader();
+ private final TableStatisticsCacheLoader tableStatisticsCacheLoader = new TableStatisticsCacheLoader();
private final AsyncLoadingCache<StatisticsCacheKey, Optional<ColumnStatistic>> columnStatisticsCache =
Caffeine.newBuilder()
@@ -78,12 +75,20 @@ public class StatisticsCache {
.executor(threadPool)
.buildAsync(histogramCacheLoader);
+ private final AsyncLoadingCache<StatisticsCacheKey, Optional<TableStatistic>> tableStatisticsCache =
+ Caffeine.newBuilder()
+ .maximumSize(Config.stats_cache_size)
+ .refreshAfterWrite(Duration.ofHours(StatisticConstants.STATISTICS_CACHE_REFRESH_INTERVAL))
+ .executor(threadPool)
+ .buildAsync(tableStatisticsCacheLoader);
+
{
threadPool.submit(() -> {
while (true) {
try {
columnStatisticsCacheLoader.removeExpiredInProgressing();
histogramCacheLoader.removeExpiredInProgressing();
+ tableStatisticsCacheLoader.removeExpiredInProgressing();
} catch (Throwable t) {
// IGNORE
}
@@ -136,6 +141,23 @@ public class StatisticsCache {
return Optional.empty();
}
+ public Optional<TableStatistic> getTableStatistics(long catalogId, long dbId, long tableId) {
+ ConnectContext ctx = ConnectContext.get();
+ if (ctx != null && ctx.getSessionVariable().internalSession) {
+ return Optional.empty();
+ }
+ StatisticsCacheKey k = new StatisticsCacheKey(catalogId, dbId, tableId);
+ try {
+ CompletableFuture<Optional<TableStatistic>> f = tableStatisticsCache.get(k);
+ if (f.isDone()) {
+ return f.get();
+ }
+ } catch (Exception e) {
+ LOG.warn("Unexpected exception while returning Histogram", e);
+ }
+ return Optional.empty();
+ }
+
public void invalidate(long tblId, long idxId, String colName) {
columnStatisticsCache.synchronous().invalidate(new StatisticsCacheKey(tblId, idxId, colName));
}
@@ -152,6 +174,14 @@ public class StatisticsCache {
columnStatisticsCache.synchronous().refresh(new StatisticsCacheKey(catalogId, dbId, tblId, idxId, colName));
}
+ public void invalidateTableStats(long catalogId, long dbId, long tblId) {
+ tableStatisticsCache.synchronous().invalidate(new StatisticsCacheKey(catalogId, dbId, tblId));
+ }
+
+ public void refreshTableStatsSync(long catalogId, long dbId, long tblId) {
+ tableStatisticsCache.synchronous().refresh(new StatisticsCacheKey(catalogId, dbId, tblId));
+ }
+
public void refreshHistogramSync(long tblId, long idxId, String colName) {
histogramCache.synchronous().refresh(new StatisticsCacheKey(tblId, idxId, colName));
}
@@ -189,113 +219,83 @@ public class StatisticsCache {
if (CollectionUtils.isEmpty(recentStatsUpdatedCols)) {
return;
}
- Map<StatisticsCacheKey, ColumnStatistic> keyToColStats = new HashMap<>();
for (ResultRow r : recentStatsUpdatedCols) {
try {
- StatsId statsId = new StatsId(r);
- long tblId = statsId.tblId;
- long idxId = statsId.idxId;
- String colId = statsId.colId;
+ long tblId = Long.parseLong(r.getColumnValue("tbl_id"));
+ long idxId = Long.parseLong(r.getColumnValue("idx_id"));
+ String colId = r.getColumnValue("col_id");
final StatisticsCacheKey k =
new StatisticsCacheKey(tblId, idxId, colId);
final ColumnStatistic c = ColumnStatistic.fromResultRow(r);
- keyToColStats.put(k, c);
+ c.loadPartitionStats(tblId, idxId, colId);
putCache(k, c);
} catch (Throwable t) {
LOG.warn("Error when preheating stats cache", t);
}
}
- try {
- loadPartStats(keyToColStats);
- } catch (Exception e) {
- LOG.warn("Fucka", e);
- }
}
- /**
- * Return false if the log of corresponding stats load is failed.
- */
- public boolean syncLoadColStats(long tableId, long idxId, String colName) {
+ public void syncLoadColStats(long tableId, long idxId, String colName) {
List<ResultRow> columnResults = StatisticsRepository.loadColStats(tableId, idxId, colName);
final StatisticsCacheKey k =
new StatisticsCacheKey(tableId, idxId, colName);
final ColumnStatistic c = ColumnStatistic.fromResultRow(columnResults);
if (c == ColumnStatistic.UNKNOWN) {
- return false;
+ return;
}
putCache(k, c);
- if (ColumnStatistic.UNKNOWN == c) {
- return false;
- }
TUpdateFollowerStatsCacheRequest updateFollowerStatsCacheRequest = new TUpdateFollowerStatsCacheRequest();
updateFollowerStatsCacheRequest.key = GsonUtils.GSON.toJson(k);
- updateFollowerStatsCacheRequest.statsRows = columnResults.stream().map(GsonUtils.GSON::toJson).collect(
- Collectors.toList());
+ updateFollowerStatsCacheRequest.colStats = GsonUtils.GSON.toJson(c);
for (Frontend frontend : Env.getCurrentEnv().getFrontends(FrontendNodeType.FOLLOWER)) {
- if (StatisticsUtil.isMaster(frontend)) {
+ if (frontend.getHost().equals(Env.getCurrentEnv().getSelfNode().getHost())) {
+ // Doesn't need to send request to current node.
continue;
}
- sendStats(frontend, updateFollowerStatsCacheRequest);
- }
- return true;
- }
-
- @VisibleForTesting
- public void sendStats(Frontend frontend, TUpdateFollowerStatsCacheRequest updateFollowerStatsCacheRequest) {
- TNetworkAddress address = new TNetworkAddress(frontend.getHost(),
- frontend.getRpcPort());
- FrontendService.Client client = null;
- try {
- client = ClientPool.frontendPool.borrowObject(address);
- client.updateStatsCache(updateFollowerStatsCacheRequest);
- } catch (Throwable t) {
- LOG.warn("Failed to sync stats to follower: {}", address, t);
- } finally {
- if (client != null) {
- ClientPool.frontendPool.returnObject(address, client);
+ TNetworkAddress address = new TNetworkAddress(frontend.getHost(),
+ frontend.getRpcPort());
+ FrontendService.Client client = null;
+ try {
+ client = ClientPool.frontendPool.borrowObject(address);
+ client.updateStatsCache(updateFollowerStatsCacheRequest);
+ } catch (Throwable t) {
+ LOG.warn("Failed to sync stats to follower: {}", address, t);
+ } finally {
+ if (client != null) {
+ ClientPool.frontendPool.returnObject(address, client);
+ }
}
}
+
}
public void putCache(StatisticsCacheKey k, ColumnStatistic c) {
- CompletableFuture<Optional<ColumnStatistic>> f = new CompletableFuture<Optional<ColumnStatistic>>();
- f.obtrudeValue(Optional.of(c));
- columnStatisticsCache.put(k, f);
- }
+ CompletableFuture<Optional<ColumnStatistic>> f = new CompletableFuture<Optional<ColumnStatistic>>() {
- private void loadPartStats(Map<StatisticsCacheKey, ColumnStatistic> keyToColStats) {
- final int batchSize = Config.expr_children_limit;
- Set<StatisticsCacheKey> keySet = new HashSet<>();
- for (StatisticsCacheKey statisticsCacheKey : keyToColStats.keySet()) {
- if (keySet.size() < batchSize - 1) {
- keySet.add(statisticsCacheKey);
- } else {
- List<ResultRow> partStats = StatisticsRepository.loadPartStats(keySet);
- addPartStatsToColStats(keyToColStats, partStats);
- keySet = new HashSet<>();
+ @Override
+ public Optional<ColumnStatistic> get() throws InterruptedException, ExecutionException {
+ return Optional.of(c);
}
- }
- if (!keySet.isEmpty()) {
- List<ResultRow> partStats = StatisticsRepository.loadPartStats(keySet);
- addPartStatsToColStats(keyToColStats, partStats);
- }
- }
- private void addPartStatsToColStats(Map<StatisticsCacheKey, ColumnStatistic> keyToColStats,
- List<ResultRow> partsStats) {
- for (ResultRow r : partsStats) {
- try {
- StatsId statsId = new StatsId(r);
- long tblId = statsId.tblId;
- long idxId = statsId.idxId;
- String partId = statsId.partId;
- String colId = statsId.colId;
- ColumnStatistic partStats = ColumnStatistic.fromResultRow(r);
- keyToColStats.get(new StatisticsCacheKey(tblId, idxId, colId)).putPartStats(partId, partStats);
- } catch (Throwable t) {
- LOG.warn("Failed to deserialized part stats", t);
+ @Override
+ public boolean isDone() {
+ return true;
}
+
+ @Override
+ public boolean complete(Optional<ColumnStatistic> value) {
+ return true;
+ }
+
+ @Override
+ public Optional<ColumnStatistic> join() {
+ return Optional.of(c);
+ }
+ };
+ if (c.isUnKnown) {
+ return;
}
+ columnStatisticsCache.put(k, f);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
index 6521a8b4a59..9aaee6bf1d7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
@@ -27,6 +27,7 @@ import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.MasterDaemon;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.statistics.util.InternalQueryResult.ResultRow;
import org.apache.doris.statistics.util.StatisticsUtil;
import org.apache.doris.system.SystemInfoService;
@@ -194,32 +195,31 @@ public class StatisticsCleaner extends MasterDaemon {
pos += StatisticConstants.FETCH_LIMIT;
for (ResultRow r : rows) {
try {
- StatsId statsId = new StatsId(r);
- String id = statsId.id;
- long catalogId = statsId.catalogId;
+ String id = r.getColumnValue("id");
+ long catalogId = Long.parseLong(r.getColumnValue("catalog_id"));
if (!idToCatalog.containsKey(catalogId)) {
expiredStats.expiredCatalog.add(catalogId);
continue;
}
- long dbId = statsId.dbId;
+ long dbId = Long.parseLong(r.getColumnValue("db_id"));
if (!idToDb.containsKey(dbId)) {
expiredStats.expiredDatabase.add(dbId);
continue;
}
- long tblId = statsId.tblId;
+ long tblId = Long.parseLong(r.getColumnValue("tbl_id"));
if (!idToTbl.containsKey(tblId)) {
expiredStats.expiredTable.add(tblId);
continue;
}
- long idxId = statsId.idxId;
+ long idxId = Long.parseLong(r.getColumnValue("idx_id"));
if (idxId != -1 && !idToMVIdx.containsKey(idxId)) {
expiredStats.expiredIdxId.add(idxId);
continue;
}
TableIf t = idToTbl.get(tblId);
- String colId = statsId.colId;
+ String colId = r.getColumnValue("col_id");
if (t.getColumn(colId) == null) {
expiredStats.ids.add(id);
continue;
@@ -228,11 +228,12 @@ public class StatisticsCleaner extends MasterDaemon {
continue;
}
OlapTable olapTable = (OlapTable) t;
- String partId = statsId.partId;
- if (partId == null) {
+ String partIdStr = r.getColumnValue("part_id");
+ if (partIdStr == null) {
continue;
}
- if (!olapTable.getPartitionIds().contains(Long.parseLong(partId))) {
+ long partId = Long.parseLong(partIdStr);
+ if (!olapTable.getPartitionIds().contains(partId)) {
expiredStats.ids.add(id);
}
} catch (Exception e) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCollector.java
deleted file mode 100644
index 2d5c4816835..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCollector.java
+++ /dev/null
@@ -1,86 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.statistics;
-
-import org.apache.doris.catalog.Env;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.common.util.MasterDaemon;
-import org.apache.doris.statistics.util.StatisticsUtil;
-
-import org.apache.hudi.common.util.VisibleForTesting;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public abstract class StatisticsCollector extends MasterDaemon {
-
- private static final Logger LOG = LogManager.getLogger(StatisticsCollector.class);
-
- protected final AnalysisTaskExecutor analysisTaskExecutor;
-
-
- public StatisticsCollector(String name, long intervalMs, AnalysisTaskExecutor analysisTaskExecutor) {
- super(name, intervalMs);
- this.analysisTaskExecutor = analysisTaskExecutor;
- analysisTaskExecutor.start();
- }
-
- @Override
- protected void runAfterCatalogReady() {
- if (!Env.getCurrentEnv().isMaster()) {
- return;
- }
- if (!StatisticsUtil.statsTblAvailable()) {
- LOG.info("Stats table not available, skip");
- return;
- }
- if (Env.isCheckpointThread()) {
- return;
- }
-
- if (!analysisTaskExecutor.idle()) {
- LOG.info("Analyze tasks those submitted in last time is not finished, skip");
- return;
- }
- collect();
- }
-
- protected abstract void collect();
-
- // Analysis job created by the system
- @VisibleForTesting
- protected void createSystemAnalysisJob(AnalysisInfo jobInfo)
- throws DdlException {
- if (jobInfo.colToPartitions.isEmpty()) {
- // No statistics need to be collected or updated
- return;
- }
-
- Map<Long, BaseAnalysisTask> analysisTaskInfos = new HashMap<>();
- AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
- analysisManager.createTaskForEachColumns(jobInfo, analysisTaskInfos, false);
- if (StatisticsUtil.isExternalTable(jobInfo.catalogName, jobInfo.dbName, jobInfo.tblName)) {
- analysisManager.createTableLevelTaskForExternalTable(jobInfo, analysisTaskInfos, false);
- }
- Env.getCurrentEnv().getAnalysisManager().registerSysJob(jobInfo, analysisTaskInfos);
- analysisTaskInfos.values().forEach(analysisTaskExecutor::submitTask);
- }
-
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsPeriodCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsPeriodCollector.java
deleted file mode 100644
index f34ad0f1221..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsPeriodCollector.java
+++ /dev/null
@@ -1,50 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.statistics;
-
-import org.apache.doris.catalog.Env;
-import org.apache.doris.common.Config;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-public class StatisticsPeriodCollector extends StatisticsCollector {
- private static final Logger LOG = LogManager.getLogger(StatisticsPeriodCollector.class);
-
- public StatisticsPeriodCollector() {
- super("Automatic Analyzer",
- TimeUnit.MINUTES.toMillis(Config.auto_check_statistics_in_minutes) / 2,
- new AnalysisTaskExecutor(Config.period_analyze_simultaneously_running_task_num));
- }
-
- @Override
- protected void collect() {
- try {
- AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
- List<AnalysisInfo> jobInfos = analysisManager.findPeriodicJobs();
- for (AnalysisInfo jobInfo : jobInfos) {
- createSystemAnalysisJob(jobInfo);
- }
- } catch (Exception e) {
- LOG.warn("Failed to periodically analyze the statistics." + e);
- }
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
index cd3cc67f3c9..d20bb358c18 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
@@ -18,6 +18,7 @@
package org.apache.doris.statistics;
import org.apache.doris.analysis.AlterColumnStatsStmt;
+import org.apache.doris.analysis.AlterTableStatsStmt;
import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
@@ -27,15 +28,16 @@ import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.statistics.util.DBObjects;
+import org.apache.doris.statistics.util.InternalQueryResult.ResultRow;
import org.apache.doris.statistics.util.StatisticsUtil;
import org.apache.doris.system.SystemInfoService;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import org.apache.commons.text.StringSubstitutor;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -93,18 +95,38 @@ public class StatisticsRepository {
+ " ORDER BY update_time "
+ "LIMIT ${limit} OFFSET ${offset}";
- private static final String FETCH_STATS_PART_ID = "SELECT * FROM "
+ private static final String FETCH_STATS_PART_ID = "SELECT col_id, part_id FROM "
+ FeConstants.INTERNAL_DB_NAME + "." + StatisticConstants.STATISTIC_TBL_NAME
+ " WHERE tbl_id = ${tblId}"
+ " AND part_id IS NOT NULL";
+ private static final String PERSIST_TABLE_STATS_TEMPLATE = "INSERT INTO "
+ + FeConstants.INTERNAL_DB_NAME + "." + StatisticConstants.ANALYSIS_TBL_NAME
+ + " VALUES('${id}', ${catalogId}, ${dbId}, ${tblId}, ${indexId}, ${partId}, ${rowCount},"
+ + " ${lastAnalyzeTimeInMs}, NOW())";
+
+ private static final String FETCH_TABLE_LEVEL_STATS_TEMPLATE = "SELECT * FROM "
+ + FeConstants.INTERNAL_DB_NAME + "." + StatisticConstants.ANALYSIS_TBL_NAME
+ + " WHERE tbl_id = ${tblId}"
+ + " AND part_id IS NULL";
+
+ private static final String FETCH_TABLE_LEVEL_PART_STATS_TEMPLATE = "SELECT * FROM "
+ + FeConstants.INTERNAL_DB_NAME + "." + StatisticConstants.ANALYSIS_TBL_NAME
+ + " WHERE part_id = ${partId}";
+
+
+ private static final String FETCH_PART_TABLE_STATS_TEMPLATE = "SELECT * FROM "
+ + FeConstants.INTERNAL_DB_NAME + "." + StatisticConstants.ANALYSIS_TBL_NAME
+ + " WHERE tbl_id = ${tblId}"
+ + " AND part_id IS NOT NULL";
+
private static final String QUERY_COLUMN_STATISTICS = "SELECT * FROM " + FeConstants.INTERNAL_DB_NAME
+ "." + StatisticConstants.STATISTIC_TBL_NAME + " WHERE "
+ "tbl_id=${tblId} AND idx_id=${idxId} AND col_id='${colId}'";
private static final String QUERY_PARTITION_STATISTICS = "SELECT * FROM " + FeConstants.INTERNAL_DB_NAME
+ "." + StatisticConstants.STATISTIC_TBL_NAME + " WHERE "
- + " ${inPredicate}"
+ + " tbl_id=${tblId} AND idx_id=${idxId} AND col_id='${colId}' "
+ " AND part_id IS NOT NULL";
public static ColumnStatistic queryColumnStatisticsByName(long tableId, String colName) {
@@ -179,7 +201,8 @@ public class StatisticsRepository {
return stringJoiner.toString();
}
- public static void dropStatistics(Set<String> partIds) throws DdlException {
+ public static void dropStatistics(Set<Long> partIds) throws DdlException {
+ dropStatisticsByPartId(partIds, StatisticConstants.ANALYSIS_TBL_NAME);
dropStatisticsByPartId(partIds, StatisticConstants.STATISTIC_TBL_NAME);
}
@@ -188,6 +211,18 @@ public class StatisticsRepository {
dropStatisticsByColName(tblId, colNames, StatisticConstants.HISTOGRAM_TBL_NAME);
}
+ public static void dropExternalTableStatistics(long tblId) throws DdlException {
+ Map<String, String> params = new HashMap<>();
+ String inPredicate = String.format("tbl_id = %s", tblId);
+ params.put("tblName", StatisticConstants.ANALYSIS_TBL_NAME);
+ params.put("condition", inPredicate);
+ try {
+ StatisticsUtil.execUpdate(new StringSubstitutor(params).replace(DROP_TABLE_STATISTICS_TEMPLATE));
+ } catch (Exception e) {
+ throw new DdlException(e.getMessage(), e);
+ }
+ }
+
public static void dropStatisticsByColName(long tblId, Set<String> colNames, String statsTblName)
throws DdlException {
Map<String, String> params = new HashMap<>();
@@ -202,7 +237,7 @@ public class StatisticsRepository {
}
}
- public static void dropStatisticsByPartId(Set<String> partIds, String statsTblName) throws DdlException {
+ public static void dropStatisticsByPartId(Set<Long> partIds, String statsTblName) throws DdlException {
Map<String, String> params = new HashMap<>();
String right = StatisticsUtil.joinElementsToString(partIds, ",");
String inPredicate = String.format(" part_id IN (%s)", right);
@@ -215,6 +250,34 @@ public class StatisticsRepository {
}
}
+ public static void persistTableStats(Map<String, String> params) throws Exception {
+ StatisticsUtil.execUpdate(PERSIST_TABLE_STATS_TEMPLATE, params);
+ }
+
+ public static void alterTableStatistics(AlterTableStatsStmt alterTableStatsStmt) throws Exception {
+ TableName tableName = alterTableStatsStmt.getTableName();
+ DBObjects objects = StatisticsUtil.convertTableNameToObjects(tableName);
+ String rowCount = alterTableStatsStmt.getValue(StatsType.ROW_COUNT);
+ TableStatisticBuilder builder = new TableStatisticBuilder();
+ builder.setRowCount(Long.parseLong(rowCount));
+ builder.setLastAnalyzeTimeInMs(0);
+ TableStatistic tableStatistic = builder.build();
+ Map<String, String> params = new HashMap<>();
+ String id = StatisticsUtil.constructId(objects.table.getId(), -1);
+ params.put("id", id);
+ params.put("catalogId", String.valueOf(objects.catalog.getId()));
+ params.put("dbId", String.valueOf(objects.db.getId()));
+ params.put("tblId", String.valueOf(objects.table.getId()));
+ params.put("indexId", "-1");
+ params.put("partId", "NULL");
+ params.put("rowCount", String.valueOf(tableStatistic.rowCount));
+ params.put("lastAnalyzeTimeInMs", "0");
+ StatisticsUtil.execUpdate(PERSIST_TABLE_STATS_TEMPLATE, params);
+ // TODO update statistics cache
+ // Env.getCurrentEnv().getStatisticsCache()
+ // .updateColStatsCache(objects.table.getId(), -1, builder.build());
+ }
+
public static void alterColumnStatistics(AlterColumnStatsStmt alterColumnStatsStmt) throws Exception {
TableName tableName = alterColumnStatsStmt.getTableName();
List<Long> partitionIds = alterColumnStatsStmt.getPartitionIds();
@@ -296,24 +359,25 @@ public class StatisticsRepository {
return StatisticsUtil.execStatisticQuery(new StringSubstitutor(params).replace(FETCH_STATS_FULL_NAME));
}
- public static Map<String, Set<String>> fetchColAndPartsForStats(long tblId) {
+ public static Map<String, Set<Long>> fetchColAndPartsForStats(long tblId) {
Map<String, String> params = Maps.newHashMap();
params.put("tblId", String.valueOf(tblId));
StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
String partSql = stringSubstitutor.replace(FETCH_STATS_PART_ID);
List<ResultRow> resultRows = StatisticsUtil.execStatisticQuery(partSql);
- Map<String, Set<String>> columnToPartitions = Maps.newHashMap();
+ Map<String, Set<Long>> columnToPartitions = Maps.newHashMap();
resultRows.forEach(row -> {
try {
- StatsId statsId = new StatsId(row);
- if (statsId.partId == null) {
+ String colId = row.getColumnValue("col_id");
+ String partId = row.getColumnValue("part_id");
+ if (partId == null) {
return;
}
- columnToPartitions.computeIfAbsent(String.valueOf(statsId.colId),
- k -> new HashSet<>()).add(statsId.partId);
- } catch (NumberFormatException e) {
+ columnToPartitions.computeIfAbsent(colId,
+ k -> new HashSet<>()).add(Long.valueOf(partId));
+ } catch (NumberFormatException | DdlException e) {
LOG.warn("Failed to obtain the column and partition for statistics.",
e);
}
@@ -322,6 +386,50 @@ public class StatisticsRepository {
return columnToPartitions;
}
+ public static TableStatistic fetchTableLevelStats(long tblId) throws DdlException {
+ ImmutableMap<String, String> params = ImmutableMap
+ .of("tblId", String.valueOf(tblId));
+ String sql = StatisticsUtil.replaceParams(FETCH_TABLE_LEVEL_STATS_TEMPLATE, params);
+ List<ResultRow> resultRows = StatisticsUtil.execStatisticQuery(sql);
+ if (resultRows.size() == 1) {
+ return TableStatistic.fromResultRow(resultRows.get(0));
+ }
+ throw new DdlException("Query result is not as expected: " + sql);
+ }
+
+ public static TableStatistic fetchTableLevelOfPartStats(long partId) throws DdlException {
+ ImmutableMap<String, String> params = ImmutableMap
+ .of("partId", String.valueOf(partId));
+ String sql = StatisticsUtil.replaceParams(FETCH_TABLE_LEVEL_PART_STATS_TEMPLATE, params);
+ List<ResultRow> resultRows = StatisticsUtil.execStatisticQuery(sql);
+ if (resultRows.size() == 1) {
+ return TableStatistic.fromResultRow(resultRows.get(0));
+ }
+ throw new DdlException("Query result is not as expected: " + sql);
+ }
+
+ public static Map<Long, TableStatistic> fetchTableLevelOfIdPartStats(long tblId) throws DdlException {
+ ImmutableMap<String, String> params = ImmutableMap
+ .of("tblId", String.valueOf(tblId));
+ StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
+ String sql = stringSubstitutor.replace(FETCH_PART_TABLE_STATS_TEMPLATE);
+ List<ResultRow> resultRows = StatisticsUtil.execStatisticQuery(sql);
+
+ if (resultRows.size() == 0) {
+ return Collections.emptyMap();
+ }
+
+ Map<Long, TableStatistic> idToPartitionTableStats = Maps.newHashMap();
+
+ for (ResultRow resultRow : resultRows) {
+ long partId = Long.parseLong(resultRow.getColumnValue("part_id"));
+ TableStatistic partStats = TableStatistic.fromResultRow(resultRow);
+ idToPartitionTableStats.put(partId, partStats);
+ }
+
+ return idToPartitionTableStats;
+ }
+
public static List<ResultRow> loadColStats(long tableId, long idxId, String colName) {
Map<String, String> params = new HashMap<>();
params.put("tblId", String.valueOf(tableId));
@@ -332,14 +440,12 @@ public class StatisticsRepository {
.replace(QUERY_COLUMN_STATISTICS));
}
- public static List<ResultRow> loadPartStats(Collection<StatisticsCacheKey> keys) {
- String inPredicate = "CONCAT(tbl_id, '-', idx_id, '-', col_id) in (%s)";
- StringJoiner sj = new StringJoiner(",");
- for (StatisticsCacheKey statisticsCacheKey : keys) {
- sj.add("'" + statisticsCacheKey.toString() + "'");
- }
+ public static List<ResultRow> loadPartStats(long tableId, long idxId, String colName) {
Map<String, String> params = new HashMap<>();
- params.put("inPredicate", String.format(inPredicate, sj.toString()));
+ params.put("tblId", String.valueOf(tableId));
+ params.put("idxId", String.valueOf(idxId));
+ params.put("colId", colName);
+
return StatisticsUtil.execStatisticQuery(new StringSubstitutor(params)
.replace(QUERY_PARTITION_STATISTICS));
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsDeriveResult.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsDeriveResult.java
index 8c301f911be..6010daa6db4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsDeriveResult.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsDeriveResult.java
@@ -144,6 +144,13 @@ public class StatsDeriveResult {
return statsDeriveResult;
}
+ public StatsDeriveResult merge(StatsDeriveResult other) {
+ for (Entry<Id, ColumnStatistic> entry : other.getSlotIdToColumnStats().entrySet()) {
+ this.slotIdToColumnStats.put(entry.getKey(), entry.getValue().copy());
+ }
+ return this;
+ }
+
public StatsDeriveResult copy() {
return new StatsDeriveResult(this);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsId.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsId.java
deleted file mode 100644
index 3f9b2641b75..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsId.java
+++ /dev/null
@@ -1,58 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.statistics;
-
-import org.apache.doris.statistics.util.StatisticsUtil;
-
-import java.util.StringJoiner;
-
-public class StatsId {
-
- public final String id;
- public final long catalogId;
- public final long dbId;
- public final long tblId;
- public final long idxId;
-
- public final String colId;
-
- // nullable
- public final String partId;
-
- public StatsId(ResultRow row) {
- this.id = row.get(0);
- this.catalogId = Long.parseLong(row.get(1));
- this.dbId = Long.parseLong(row.get(2));
- this.tblId = Long.parseLong(row.get(3));
- this.idxId = Long.parseLong(row.get(4));
- this.colId = row.get(5);
- this.partId = row.get(6);
- }
-
- public String toSQL() {
- StringJoiner sj = new StringJoiner(",");
- sj.add(StatisticsUtil.quote(id));
- sj.add(String.valueOf(catalogId));
- sj.add(String.valueOf(dbId));
- sj.add(String.valueOf(tblId));
- sj.add(String.valueOf(idxId));
- sj.add(StatisticsUtil.quote(colId));
- sj.add(StatisticsUtil.quote(partId));
- return sj.toString();
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatistic.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatistic.java
new file mode 100644
index 00000000000..28d0c17b561
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatistic.java
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.statistics;
+
+import org.apache.doris.common.DdlException;
+import org.apache.doris.statistics.util.InternalQueryResult.ResultRow;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class TableStatistic {
+
+ private static final Logger LOG = LogManager.getLogger(TableStatistic.class);
+
+ public static TableStatistic UNKNOWN = new TableStatisticBuilder()
+ .setRowCount(0).setUpdateTime("NULL").setLastAnalyzeTimeInMs(0L)
+ .build();
+
+ public final long rowCount;
+ public final long lastAnalyzeTimeInMs;
+ public final String updateTime;
+
+ public TableStatistic(long rowCount, long lastAnalyzeTimeInMs, String updateTime) {
+ this.rowCount = rowCount;
+ this.lastAnalyzeTimeInMs = lastAnalyzeTimeInMs;
+ this.updateTime = updateTime;
+ }
+
+ // TODO: use thrift
+ public static TableStatistic fromResultRow(ResultRow resultRow) {
+ try {
+ TableStatisticBuilder tableStatisticBuilder = new TableStatisticBuilder();
+ long rowCount = Long.parseLong(resultRow.getColumnValue("count"));
+ String updateTime = resultRow.getColumnValue("update_time");
+ long lastAnalyzeTimeInMs = Long
+ .parseLong(resultRow.getColumnValue("last_analyze_time_in_ms"));
+ tableStatisticBuilder.setRowCount(rowCount);
+ tableStatisticBuilder.setLastAnalyzeTimeInMs(lastAnalyzeTimeInMs);
+ tableStatisticBuilder.setUpdateTime(updateTime);
+ return tableStatisticBuilder.build();
+ } catch (DdlException e) {
+ LOG.warn("Failed to deserialize table statistics", e);
+ return TableStatistic.UNKNOWN;
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatisticBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatisticBuilder.java
new file mode 100644
index 00000000000..ddb45b824cb
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatisticBuilder.java
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.statistics;
+
+public class TableStatisticBuilder {
+ public long rowCount;
+ public long lastAnalyzeTimeInMs;
+ public String updateTime;
+
+ public TableStatisticBuilder() {
+ }
+
+ public TableStatisticBuilder(TableStatistic tableStatistic) {
+ this.rowCount = tableStatistic.rowCount;
+ this.updateTime = tableStatistic.updateTime;
+ }
+
+ public TableStatisticBuilder setRowCount(long rowCount) {
+ this.rowCount = rowCount;
+ return this;
+ }
+
+ public TableStatisticBuilder setLastAnalyzeTimeInMs(long lastAnalyzeTimeInMs) {
+ this.lastAnalyzeTimeInMs = lastAnalyzeTimeInMs;
+ return this;
+ }
+
+ public TableStatisticBuilder setUpdateTime(String updateTime) {
+ this.updateTime = updateTime;
+ return this;
+ }
+
+ public TableStatistic build() {
+ return new TableStatistic(rowCount, lastAnalyzeTimeInMs, updateTime);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatisticsCacheLoader.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatisticsCacheLoader.java
new file mode 100644
index 00000000000..953bc9a4274
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatisticsCacheLoader.java
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.statistics;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.common.DdlException;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Optional;
+
+public class TableStatisticsCacheLoader extends StatisticsCacheLoader<Optional<TableStatistic>> {
+
+ private static final Logger LOG = LogManager.getLogger(TableStatisticsCacheLoader.class);
+
+ @Override
+ protected Optional<TableStatistic> doLoad(StatisticsCacheKey key) {
+ try {
+ TableStatistic tableStatistic = StatisticsRepository.fetchTableLevelStats(key.tableId);
+ if (tableStatistic != TableStatistic.UNKNOWN) {
+ return Optional.of(tableStatistic);
+ }
+ } catch (DdlException e) {
+ LOG.debug("Fail to get table line number from table_statistics table. "
+ + "Will try to get from data source.", e);
+ }
+ // Get row count by call TableIf interface getRowCount
+ // when statistic table doesn't contain a record for this table.
+ try {
+ TableIf table = Env.getCurrentEnv().getCatalogMgr().getCatalog(key.catalogId)
+ .getDbOrDdlException(key.dbId).getTableOrAnalysisException(key.tableId);
+ long rowCount = table.getRowCount();
+ long lastAnalyzeTimeInMs = System.currentTimeMillis();
+ String updateTime = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(lastAnalyzeTimeInMs));
+ return Optional.of(new TableStatistic(rowCount, lastAnalyzeTimeInMs, updateTime));
+ } catch (Exception e) {
+ LOG.warn(String.format("Fail to get row count for table %d", key.tableId), e);
+ }
+ return Optional.empty();
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
deleted file mode 100644
index 17ca61e9da5..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
+++ /dev/null
@@ -1,137 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.statistics;
-
-import org.apache.doris.common.io.Text;
-import org.apache.doris.common.io.Writable;
-import org.apache.doris.persist.gson.GsonUtils;
-import org.apache.doris.statistics.AnalysisInfo.JobType;
-
-import com.google.gson.annotations.SerializedName;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.stream.Collectors;
-
-public class TableStatsMeta implements Writable {
-
- @SerializedName("tblId")
- public final long tblId;
-
- @SerializedName("idxId")
- public final long idxId;
- @SerializedName("updatedRows")
- public final AtomicLong updatedRows = new AtomicLong();
-
- // We would like to analyze tables which queried frequently with higher priority in the future.
- @SerializedName("queriedTimes")
- public final AtomicLong queriedTimes = new AtomicLong();
-
- // Used for external table.
- @SerializedName("rowCount")
- public final long rowCount;
-
- @SerializedName("updateTime")
- public long updatedTime;
-
- @SerializedName("colNameToColStatsMeta")
- private ConcurrentMap<String, ColStatsMeta> colNameToColStatsMeta = new ConcurrentHashMap<>();
-
- @SerializedName("trigger")
- public JobType jobType;
-
- // It's necessary to store these fields separately from AnalysisInfo, since the lifecycle between AnalysisInfo
- // and TableStats is quite different.
- public TableStatsMeta(long tblId, long rowCount, AnalysisInfo analyzedJob) {
- this.tblId = tblId;
- this.idxId = -1;
- this.rowCount = rowCount;
- updateByJob(analyzedJob);
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- String json = GsonUtils.GSON.toJson(this);
- Text.writeString(out, json);
- }
-
- public static TableStatsMeta read(DataInput dataInput) throws IOException {
- String json = Text.readString(dataInput);
- TableStatsMeta tableStats = GsonUtils.GSON.fromJson(json, TableStatsMeta.class);
- // Might be null counterintuitively, for compatible
- if (tableStats.colNameToColStatsMeta == null) {
- tableStats.colNameToColStatsMeta = new ConcurrentHashMap<>();
- }
- return tableStats;
- }
-
- public long findColumnLastUpdateTime(String colName) {
- ColStatsMeta colStatsMeta = colNameToColStatsMeta.get(colName);
- if (colStatsMeta == null) {
- return 0;
- }
- return colStatsMeta.updatedTime;
- }
-
- public ColStatsMeta findColumnStatsMeta(String colName) {
- return colNameToColStatsMeta.get(colName);
- }
-
- public void removeColumn(String colName) {
- colNameToColStatsMeta.remove(colName);
- }
-
- public Set<String> analyzeColumns() {
- return colNameToColStatsMeta.keySet();
- }
-
- public void reset() {
- updatedTime = 0;
- colNameToColStatsMeta.values().forEach(ColStatsMeta::clear);
- }
-
- public void updateByJob(AnalysisInfo analyzedJob) {
- updatedTime = System.currentTimeMillis();
- String colNameStr = analyzedJob.colName;
- // colName field AnalyzeJob's format likes: "[col1, col2]", we need to remove brackets here
- // TODO: Refactor this later
- if (analyzedJob.colName.startsWith("[") && analyzedJob.colName.endsWith("]")) {
- colNameStr = colNameStr.substring(1, colNameStr.length() - 1);
- }
- List<String> cols = Arrays.stream(colNameStr.split(",")).map(String::trim).collect(Collectors.toList());
- for (String col : cols) {
- ColStatsMeta colStatsMeta = colNameToColStatsMeta.get(col);
- if (colStatsMeta == null) {
- colNameToColStatsMeta.put(col, new ColStatsMeta(updatedTime,
- analyzedJob.analysisMethod, analyzedJob.analysisType, analyzedJob.jobType, 0));
- } else {
- colStatsMeta.updatedTime = updatedTime;
- colStatsMeta.analysisType = analyzedJob.analysisType;
- colStatsMeta.analysisMethod = analyzedJob.analysisMethod;
- }
- }
- jobType = analyzedJob.jobType;
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/TaskStatusWrapper.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/TaskStatusWrapper.java
deleted file mode 100644
index d74b14267d1..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/TaskStatusWrapper.java
+++ /dev/null
@@ -1,33 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.statistics;
-
-public class TaskStatusWrapper {
-
- public final AnalysisInfo info;
- public final AnalysisState taskState;
- public final String message;
- public final long time;
-
- public TaskStatusWrapper(AnalysisInfo info, AnalysisState taskState, String message, long time) {
- this.info = info;
- this.taskState = taskState;
- this.message = message;
- this.time = time;
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/InternalQuery.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/InternalQuery.java
index 40669b6a939..09af38d830a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/InternalQuery.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/InternalQuery.java
@@ -24,6 +24,7 @@ import org.apache.doris.analysis.SqlScanner;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
@@ -37,7 +38,7 @@ import org.apache.doris.qe.Coordinator;
import org.apache.doris.qe.OriginStatement;
import org.apache.doris.qe.QeProcessorImpl;
import org.apache.doris.qe.RowBatch;
-import org.apache.doris.statistics.ResultRow;
+import org.apache.doris.statistics.util.InternalQueryResult.ResultRow;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.TQueryOptions;
import org.apache.doris.thrift.TResultBatch;
@@ -49,9 +50,9 @@ import org.apache.logging.log4j.Logger;
import java.io.StringReader;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
+import java.util.stream.Collectors;
/**
* Execute SQL query statements internally(in FE). Internal-query mainly used for statistics module,
@@ -86,7 +87,7 @@ public class InternalQuery {
* @return Result of the query statement
* @throws Exception Errors in parsing or execution
*/
- public List<ResultRow> query() throws Exception {
+ public InternalQueryResult query() throws Exception {
// step1: mock connectContext
buildContext();
@@ -179,9 +180,14 @@ public class InternalQuery {
}
}
- private List<ResultRow> fetchResult() {
+ private InternalQueryResult fetchResult() {
List<String> columns = stmt.getColLabels();
- List<ResultRow> resultRows = new ArrayList<>();
+ List<PrimitiveType> types = stmt.getResultExprs().stream()
+ .map(e -> e.getType().getPrimitiveType())
+ .collect(Collectors.toList());
+
+ InternalQueryResult result = new InternalQueryResult();
+ List<ResultRow> resultRows = result.getResultRows();
for (TResultBatch batch : resultBatches) {
List<ByteBuffer> rows = batch.getRows();
@@ -194,11 +200,12 @@ public class InternalQuery {
values.add(value);
}
- ResultRow resultRow = new ResultRow(values);
+ ResultRow resultRow = new ResultRow(columns, types, values);
resultRows.add(resultRow);
}
}
- return resultRows;
+
+ return result;
}
public void cancel() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/InternalQueryResult.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/InternalQueryResult.java
new file mode 100644
index 00000000000..e7919860107
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/InternalQueryResult.java
@@ -0,0 +1,242 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.statistics.util;
+
+import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.common.DdlException;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Readable results of internal SQL execution,
+ * providing some read operations.
+ */
+public class InternalQueryResult {
+ private final List<ResultRow> resultRows = Lists.newArrayList();
+
+ public InternalQueryResult() {
+ }
+
+ public List<ResultRow> getResultRows() {
+ return resultRows;
+ }
+
+ public static class ResultRow {
+ private final List<String> columns;
+ private final List<PrimitiveType> types;
+ private final List<String> values;
+
+ private final Map<String, Integer> columnNameMap = Maps.newHashMap();
+ private final Map<Integer, String> columnIndexMap = Maps.newHashMap();
+
+ public ResultRow(List<String> columns, List<PrimitiveType> types, List<String> values) {
+ this.columns = columns;
+ this.types = types;
+ this.values = values;
+ buildColumnNameMap();
+ buildColumnIndexMap();
+ }
+
+ public List<String> getColumns() {
+ return columns != null ? columns : Collections.emptyList();
+ }
+
+ public List<PrimitiveType> getTypes() {
+ return types != null ? types : Collections.emptyList();
+ }
+
+ public List<String> getValues() {
+ return values != null ? values : Collections.emptyList();
+ }
+
+ private void buildColumnNameMap() {
+ List<String> columns = getColumns();
+ for (int i = 0; i < columns.size(); i++) {
+ columnNameMap.put(columns.get(i), i);
+ }
+ }
+
+ private void buildColumnIndexMap() {
+ List<String> columns = getColumns();
+ for (int i = 0; i < columns.size(); i++) {
+ columnIndexMap.put(i, columns.get(i));
+ }
+ }
+
+ public int getColumnIndex(String columnName) {
+ return columnNameMap.getOrDefault(columnName, -1);
+ }
+
+ public String getColumnName(int index) throws DdlException {
+ List<String> columns = getColumns();
+ if (columnIndexMap.containsKey(index)) {
+ return columnIndexMap.get(index);
+ } else {
+ throw new DdlException("Index should be between 0 and " + columns.size());
+ }
+ }
+
+ public PrimitiveType getColumnType(String columnName) throws DdlException {
+ List<PrimitiveType> types = getTypes();
+ int index = getColumnIndex(columnName);
+ if (index == -1) {
+ throw new DdlException(String.format("The column name:[%s] does not exist.", columnName));
+ }
+ return types.get(index);
+ }
+
+ public PrimitiveType getColumnType(int index) throws DdlException {
+ List<PrimitiveType> types = getTypes();
+ if (index >= 0 && index < types.size()) {
+ return types.get(index);
+ } else {
+ throw new DdlException("Index should be between 0 and " + types.size());
+ }
+ }
+
+ public String getColumnValue(String columnName) throws DdlException {
+ int index = getColumnIndex(columnName);
+ if (index == -1) {
+ throw new DdlException(String.format("The column name:[%s] does not exist.", columnName));
+ }
+ return values.get(index);
+ }
+
+ public String getColumnValueWithDefault(String columnName, String defaultVal) throws DdlException {
+ String val = getColumnValue(columnName);
+ return val == null ? defaultVal : val;
+ }
+
+ public Object getColumnValue(int index) throws DdlException {
+ List<String> columns = getColumns();
+ if (index >= 0 && index < columns.size()) {
+ return values.get(index);
+ } else {
+ throw new DdlException("Index should be between 0 and " + columns.size());
+ }
+ }
+
+ public String getString(int index) throws DdlException {
+ List<String> columns = getColumns();
+ if (index >= 0 && index < columns.size()) {
+ return values.get(index);
+ }
+ throw new DdlException("Index should be between 0 and " + columns.size());
+ }
+
+ public int getInt(int index) throws DdlException {
+ List<PrimitiveType> types = getTypes();
+ if (index >= 0 && index < types.size()) {
+ String value = values.get(index);
+ PrimitiveType type = types.get(index);
+ switch (type) {
+ case BOOLEAN:
+ case TINYINT:
+ case SMALLINT:
+ case INT:
+ case BIGINT:
+ return new Integer(value);
+ default:
+ throw new DdlException("Unable to convert field to int: " + value);
+ }
+ }
+ throw new DdlException("Index should be between 0 and " + types.size());
+ }
+
+ public long getLong(int index) throws DdlException {
+ List<PrimitiveType> types = getTypes();
+ if (index >= 0 && index < types.size()) {
+ String value = values.get(index);
+ PrimitiveType type = types.get(index);
+ switch (type) {
+ case TINYINT:
+ case SMALLINT:
+ case INT:
+ case BIGINT:
+ return Long.parseLong(value);
+ default:
+ throw new DdlException("Unable to convert field to long: " + value);
+ }
+ }
+ throw new DdlException("Index should be between 0 and " + types.size());
+ }
+
+ public float getFloat(int index) throws DdlException {
+ List<PrimitiveType> types = getTypes();
+ if (index >= 0 && index < types.size()) {
+ String value = values.get(index);
+ PrimitiveType type = types.get(index);
+ if (type == PrimitiveType.FLOAT) {
+ return Float.parseFloat(value);
+ }
+ throw new DdlException("Unable to convert field to float: " + value);
+ }
+ throw new DdlException("Index should be between 0 and " + types.size());
+ }
+
+ public double getDouble(int index) throws DdlException {
+ List<PrimitiveType> types = getTypes();
+ if (index >= 0 && index < types.size()) {
+ String value = values.get(index);
+ PrimitiveType type = types.get(index);
+ if (type == PrimitiveType.DOUBLE) {
+ return Double.parseDouble(value);
+ }
+ throw new DdlException("Unable to convert field to long: " + value);
+ }
+ throw new DdlException("Index should be between 0 and " + types.size());
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("ResultRow{ ");
+ if (values != null && values.size() > 0) {
+ List<String> columns = getColumns();
+ for (int i = 0; i < values.size(); i++) {
+ sb.append(columns.get(i));
+ sb.append(":");
+ sb.append(values.get(i));
+ sb.append(" ");
+ }
+ }
+ sb.append("}");
+ return sb.toString();
+ }
+ }
+
+ @Override
+ public String toString() {
+ if (resultRows.size() > 0) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("InternalQueryResult:\n");
+ for (ResultRow resultRow : resultRows) {
+ sb.append(" - ");
+ sb.append(resultRow.toString());
+ sb.append("\n");
+ }
+ return sb.toString();
+ }
+ return "InternalQueryResult{" + "resultRows=" + resultRows + '}';
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/SimpleQueue.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/SimpleQueue.java
deleted file mode 100644
index 5740c4e3088..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/SimpleQueue.java
+++ /dev/null
@@ -1,65 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.statistics.util;
-
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.function.Function;
-
-// Any operation on this structure should be thread-safe
-public class SimpleQueue<T> extends LinkedList<T> {
-
- private final long limit;
-
- private final Function<T, Void> offerFunc;
-
- private final Function<T, Void> evictFunc;
-
-
- public SimpleQueue(long limit, Function<T, Void> offerFunc, Function<T, Void> evictFunc) {
- this.limit = limit;
- this.offerFunc = offerFunc;
- this.evictFunc = evictFunc;
- }
-
- @Override
- public synchronized boolean offer(T analysisInfo) {
- while (size() >= limit) {
- remove();
- }
- super.offer(analysisInfo);
- offerFunc.apply(analysisInfo);
- return true;
- }
-
- @Override
- public synchronized T remove() {
- T analysisInfo = super.remove();
- evictFunc.apply(analysisInfo);
- return analysisInfo;
- }
-
- public SimpleQueue(long limit, Function<T, Void> offerFunc, Function<T, Void> evictFunc, Collection<T> collection) {
- this(limit, offerFunc, evictFunc);
- if (collection != null) {
- for (T e : collection) {
- offer(e);
- }
- }
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index 40ae13a0e0e..89a5ae1f3e6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -25,12 +25,10 @@ import org.apache.doris.analysis.FloatLiteral;
import org.apache.doris.analysis.IntLiteral;
import org.apache.doris.analysis.LargeIntLiteral;
import org.apache.doris.analysis.LiteralExpr;
-import org.apache.doris.analysis.SetType;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.analysis.TableName;
import org.apache.doris.analysis.UserIdentity;
-import org.apache.doris.analysis.VariableExpr;
import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DatabaseIf;
@@ -46,12 +44,10 @@ import org.apache.doris.catalog.StructType;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.Type;
import org.apache.doris.catalog.VariantType;
-import org.apache.doris.catalog.external.ExternalTable;
import org.apache.doris.catalog.external.HMSExternalTable;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.FeConstants;
-import org.apache.doris.common.Pair;
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.HMSExternalCatalog;
@@ -65,13 +61,12 @@ import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.QueryState;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.qe.StmtExecutor;
-import org.apache.doris.qe.VariableMgr;
+import org.apache.doris.statistics.AnalysisInfo;
import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.ColumnStatisticBuilder;
import org.apache.doris.statistics.Histogram;
-import org.apache.doris.statistics.ResultRow;
import org.apache.doris.statistics.StatisticConstants;
-import org.apache.doris.system.Frontend;
+import org.apache.doris.statistics.util.InternalQueryResult.ResultRow;
import org.apache.doris.system.SystemInfoService;
import com.google.common.base.Preconditions;
@@ -87,12 +82,9 @@ import org.apache.iceberg.TableScan;
import org.apache.iceberg.types.Types;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import org.apache.thrift.TException;
-import java.net.InetSocketAddress;
import java.text.SimpleDateFormat;
-import java.time.LocalTime;
-import java.time.format.DateTimeFormatter;
-import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -102,7 +94,6 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
-import java.util.Set;
import java.util.StringJoiner;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -111,6 +102,7 @@ public class StatisticsUtil {
private static final Logger LOG = LogManager.getLogger(StatisticsUtil.class);
private static final String ID_DELIMITER = "-";
+ private static final String VALUES_DELIMITER = ",";
private static final String TOTAL_SIZE = "totalSize";
private static final String NUM_ROWS = "numRows";
@@ -150,6 +142,16 @@ public class StatisticsUtil {
}
}
+ public static List<AnalysisInfo> deserializeToAnalysisJob(List<ResultRow> resultBatches)
+ throws TException {
+ if (CollectionUtils.isEmpty(resultBatches)) {
+ return Collections.emptyList();
+ }
+ return resultBatches.stream()
+ .map(AnalysisInfo::fromResultRow)
+ .collect(Collectors.toList());
+ }
+
public static ColumnStatistic deserializeToColumnStatistics(List<ResultRow> resultBatches)
throws Exception {
if (CollectionUtils.isEmpty(resultBatches)) {
@@ -164,22 +166,15 @@ public class StatisticsUtil {
}
public static AutoCloseConnectContext buildConnectContext() {
- return buildConnectContext(false);
- }
-
- public static AutoCloseConnectContext buildConnectContext(boolean limitScan) {
ConnectContext connectContext = new ConnectContext();
SessionVariable sessionVariable = connectContext.getSessionVariable();
sessionVariable.internalSession = true;
sessionVariable.setMaxExecMemByte(Config.statistics_sql_mem_limit_in_bytes);
- sessionVariable.cpuResourceLimit = Config.cpu_resource_limit_per_analyze_task;
sessionVariable.setEnableInsertStrict(true);
- sessionVariable.enablePageCache = false;
sessionVariable.parallelExecInstanceNum = Config.statistics_sql_parallel_exec_instance_num;
sessionVariable.parallelPipelineTaskNum = Config.statistics_sql_parallel_exec_instance_num;
sessionVariable.setEnableNereidsPlanner(false);
sessionVariable.enableProfile = false;
- sessionVariable.enableScanRunSerial = limitScan;
sessionVariable.queryTimeoutS = Config.analyze_task_timeout_in_hours * 60 * 60;
sessionVariable.insertTimeoutS = Config.analyze_task_timeout_in_hours * 60 * 60;
sessionVariable.enableFileCache = false;
@@ -221,7 +216,7 @@ public class StatisticsUtil {
case DOUBLE:
return new FloatLiteral(columnValue);
case DECIMALV2:
- // no need to check precision and scale, since V2 is fixed point
+ //no need to check precision and scale, since V2 is fixed point
return new DecimalLiteral(columnValue);
case DECIMAL32:
case DECIMAL64:
@@ -397,12 +392,11 @@ public class StatisticsUtil {
.findTable(InternalCatalog.INTERNAL_CATALOG_NAME,
dbName,
StatisticConstants.STATISTIC_TBL_NAME));
- // uncomment it when hist is available for user.
- // statsTbls.add(
- // (OlapTable) StatisticsUtil
- // .findTable(InternalCatalog.INTERNAL_CATALOG_NAME,
- // dbName,
- // StatisticConstants.HISTOGRAM_TBL_NAME));
+ statsTbls.add(
+ (OlapTable) StatisticsUtil
+ .findTable(InternalCatalog.INTERNAL_CATALOG_NAME,
+ dbName,
+ StatisticConstants.HISTOGRAM_TBL_NAME));
} catch (Throwable t) {
return false;
}
@@ -436,15 +430,6 @@ public class StatisticsUtil {
));
}
- public static Set<String> getPartitionIds(TableIf table) {
- if (table instanceof OlapTable) {
- return ((OlapTable) table).getPartitionIds().stream().map(String::valueOf).collect(Collectors.toSet());
- } else if (table instanceof ExternalTable) {
- return table.getPartitionNames();
- }
- throw new RuntimeException(String.format("Not supported Table %s", table.getClass().getName()));
- }
-
public static <T> String joinElementsToString(Collection<T> values, String delimiter) {
StringJoiner builder = new StringJoiner(delimiter);
values.forEach(v -> builder.add(String.valueOf(v)));
@@ -490,9 +475,9 @@ public class StatisticsUtil {
* when update_rows < row_count, the health degree is 100 (1 - update_rows row_count).
*
* @param updatedRows The number of rows updated by the table
- * @param totalRows The current number of rows in the table
- * the healthier the statistics of the table
* @return Health, the value range is [0, 100], the larger the value,
+ * @param totalRows The current number of rows in the table
+ * the healthier the statistics of the table
*/
public static int getTableHealth(long totalRows, long updatedRows) {
if (updatedRows >= totalRows) {
@@ -506,25 +491,19 @@ public class StatisticsUtil {
/**
* Estimate hive table row count.
* First get it from remote table parameters. If not found, estimate it : totalSize/estimatedRowSize
- *
* @param table Hive HMSExternalTable to estimate row count.
- * @param isInit Flag to indicate if this is called during init. To avoid recursively get schema.
* @return estimated row count
*/
- public static long getHiveRowCount(HMSExternalTable table, boolean isInit) {
+ public static long getHiveRowCount(HMSExternalTable table) {
Map<String, String> parameters = table.getRemoteTable().getParameters();
if (parameters == null) {
return -1;
}
// Table parameters contains row count, simply get and return it.
if (parameters.containsKey(NUM_ROWS)) {
- long rows = Long.parseLong(parameters.get(NUM_ROWS));
- // Sometimes, the NUM_ROWS in hms is 0 but actually is not. Need to check TOTAL_SIZE if NUM_ROWS is 0.
- if (rows != 0) {
- return rows;
- }
+ return Long.parseLong(parameters.get(NUM_ROWS));
}
- if (!parameters.containsKey(TOTAL_SIZE) || isInit) {
+ if (!parameters.containsKey(TOTAL_SIZE)) {
return -1;
}
// Table parameters doesn't contain row count but contain total size. Estimate row count : totalSize/rowSize
@@ -542,7 +521,6 @@ public class StatisticsUtil {
/**
* Estimate iceberg table row count.
* Get the row count by adding all task file recordCount.
- *
* @param table Iceberg HMSExternalTable to estimate row count.
* @return estimated row count
*/
@@ -566,7 +544,6 @@ public class StatisticsUtil {
/**
* Estimate hive table row count : totalFileSize/estimatedRowSize
- *
* @param table Hive HMSExternalTable to estimate row count.
* @return estimated row count
*/
@@ -641,7 +618,6 @@ public class StatisticsUtil {
/**
* Get Iceberg column statistics.
- *
* @param colName
* @param table Iceberg table.
* @return Optional Column statistic for the given column.
@@ -650,8 +626,8 @@ public class StatisticsUtil {
TableScan tableScan = table.newScan().includeColumnStats();
ColumnStatisticBuilder columnStatisticBuilder = new ColumnStatisticBuilder();
columnStatisticBuilder.setCount(0);
- columnStatisticBuilder.setMaxValue(Double.POSITIVE_INFINITY);
- columnStatisticBuilder.setMinValue(Double.NEGATIVE_INFINITY);
+ columnStatisticBuilder.setMaxValue(Double.MAX_VALUE);
+ columnStatisticBuilder.setMinValue(Double.MIN_VALUE);
columnStatisticBuilder.setDataSize(0);
columnStatisticBuilder.setAvgSizeByte(0);
columnStatisticBuilder.setNumNulls(0);
@@ -666,7 +642,7 @@ public class StatisticsUtil {
}
private static void processDataFile(DataFile dataFile, PartitionSpec partitionSpec,
- String colName, ColumnStatisticBuilder columnStatisticBuilder) {
+ String colName, ColumnStatisticBuilder columnStatisticBuilder) {
int colId = -1;
for (Types.NestedField column : partitionSpec.schema().columns()) {
if (column.name().equals(colName)) {
@@ -702,87 +678,4 @@ public class StatisticsUtil {
// IGNORE
}
}
-
- public static String quote(String str) {
- return "'" + str + "'";
- }
-
- public static boolean isMaster(Frontend frontend) {
- InetSocketAddress socketAddress = new InetSocketAddress(frontend.getHost(), frontend.getEditLogPort());
- return Env.getCurrentEnv().getHaProtocol().getLeader().equals(socketAddress);
- }
-
- public static String escapeSQL(String str) {
- if (str == null) {
- return null;
- }
- return org.apache.commons.lang3.StringUtils.replace(str, "'", "''");
- }
-
- public static boolean isExternalTable(String catalogName, String dbName, String tblName) {
- TableIf table;
- try {
- table = StatisticsUtil.findTable(catalogName, dbName, tblName);
- } catch (Throwable e) {
- LOG.warn(e.getMessage());
- return false;
- }
- return table instanceof ExternalTable;
- }
-
- public static boolean inAnalyzeTime(LocalTime now) {
- try {
- Pair<LocalTime, LocalTime> range = findRangeFromGlobalSessionVar();
- if (range == null) {
- return false;
- }
- LocalTime start = range.first;
- LocalTime end = range.second;
- if (start.isAfter(end) && (now.isAfter(start) || now.isBefore(end))) {
- return true;
- } else {
- return now.isAfter(start) && now.isBefore(end);
- }
- } catch (DateTimeParseException e) {
- LOG.warn("Parse analyze start/end time format fail", e);
- return true;
- }
- }
-
- private static Pair<LocalTime, LocalTime> findRangeFromGlobalSessionVar() {
- try {
- String startTime =
- findRangeFromGlobalSessionVar(SessionVariable.FULL_AUTO_ANALYZE_START_TIME)
- .fullAutoAnalyzeStartTime;
- // For compatibility
- if (StringUtils.isEmpty(startTime)) {
- startTime = StatisticConstants.FULL_AUTO_ANALYZE_START_TIME;
- }
- String endTime = findRangeFromGlobalSessionVar(SessionVariable.FULL_AUTO_ANALYZE_END_TIME)
- .fullAutoAnalyzeEndTime;
- if (StringUtils.isEmpty(startTime)) {
- endTime = StatisticConstants.FULL_AUTO_ANALYZE_END_TIME;
- }
- DateTimeFormatter timeFormatter = DateTimeFormatter.ofPattern("HH:mm:ss");
- return Pair.of(LocalTime.parse(startTime, timeFormatter), LocalTime.parse(endTime, timeFormatter));
- } catch (Exception e) {
- return null;
- }
- }
-
- private static SessionVariable findRangeFromGlobalSessionVar(String varName) throws Exception {
- SessionVariable sessionVariable = VariableMgr.newSessionVariable();
- VariableExpr variableExpr = new VariableExpr(varName, SetType.GLOBAL);
- VariableMgr.getValue(sessionVariable, variableExpr);
- return sessionVariable;
- }
-
- public static boolean enableAutoAnalyze() {
- try {
- return findRangeFromGlobalSessionVar(SessionVariable.ENABLE_FULL_AUTO_ANALYZE).enableFullAutoAnalyze;
- } catch (Exception e) {
- LOG.warn("Fail to get value of enable auto analyze, return false by default", e);
- }
- return false;
- }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/OtherJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/OtherJoinTest.java
index a4062d2edff..feeb971b15e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/OtherJoinTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/OtherJoinTest.java
@@ -20,7 +20,6 @@ package org.apache.doris.nereids.jobs.joinorder.hypergraph;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.datasets.tpch.TPCHTestBase;
import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.util.HyperGraphBuilder;
import org.apache.doris.nereids.util.MemoTestUtils;
import org.apache.doris.nereids.util.PlanChecker;
@@ -33,37 +32,23 @@ import java.util.Set;
public class OtherJoinTest extends TPCHTestBase {
@Test
- public void test() {
- for (int t = 3; t < 10; t++) {
- for (int e = t - 1; e <= (t * (t - 1)) / 2; e++) {
- for (int i = 0; i < 10; i++) {
- System.out.println(String.valueOf(t) + " " + e + ": " + i);
- randomTest(t, e);
- }
- }
- }
- }
-
- private void randomTest(int tableNum, int edgeNum) {
+ public void randomTest() {
HyperGraphBuilder hyperGraphBuilder = new HyperGraphBuilder();
Plan plan = hyperGraphBuilder
- .randomBuildPlanWith(tableNum, edgeNum);
- plan = new LogicalProject(plan.getOutput(), plan);
- Set<List<String>> res1 = hyperGraphBuilder.evaluate(plan);
+ .randomBuildPlanWith(10, 20);
+ Set<List<Integer>> res1 = hyperGraphBuilder.evaluate(plan);
CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(connectContext, plan);
hyperGraphBuilder.initStats(cascadesContext);
Plan optimizedPlan = PlanChecker.from(cascadesContext)
- .dpHypOptimize()
- .getBestPlanTree();
+ .dpHypOptimize()
+ .getBestPlanTree();
- Set<List<String>> res2 = hyperGraphBuilder.evaluate(optimizedPlan);
+ Set<List<Integer>> res2 = hyperGraphBuilder.evaluate(optimizedPlan);
if (!res1.equals(res2)) {
- System.out.println(plan.treeString());
- System.out.println(optimizedPlan.treeString());
- cascadesContext = MemoTestUtils.createCascadesContext(connectContext, plan);
- PlanChecker.from(cascadesContext).dpHypOptimize().getBestPlanTree();
System.out.println(res1);
System.out.println(res2);
+ System.out.println(plan.treeString());
+ System.out.println(optimizedPlan.treeString());
}
Assertions.assertTrue(res1.equals(res2));
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java
index 31affe06252..1fe5e5b0a0e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java
@@ -17,7 +17,6 @@
package org.apache.doris.nereids.stats;
-import org.apache.doris.analysis.IntLiteral;
import org.apache.doris.nereids.trees.expressions.And;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.EqualTo;
@@ -76,7 +75,7 @@ class FilterEstimationTest {
Statistics expected = filterEstimation.estimate(or, stat);
Assertions.assertTrue(
Precision.equals(expected.getRowCount(), 750,
- 0.01));
+ 0.01));
}
// a > 500 and b < 100
@@ -133,12 +132,12 @@ class FilterEstimationTest {
Map<Expression, ColumnStatistic> slotToColumnStat = new HashMap<>();
ColumnStatisticBuilder builder = new ColumnStatisticBuilder()
.setNdv(500)
- .setIsUnknown(false);
+ .setIsUnknown(true);
slotToColumnStat.put(a, builder.build());
Statistics stat = new Statistics(1000, slotToColumnStat);
FilterEstimation filterEstimation = new FilterEstimation();
Statistics expected = filterEstimation.estimate(notIn, stat);
- Assertions.assertTrue(Precision.equals(1000, expected.getRowCount(), 0.01));
+ Assertions.assertTrue(Precision.equals(666.666, expected.getRowCount(), 0.01));
}
/**
@@ -166,6 +165,7 @@ class FilterEstimationTest {
ColumnStatistic aStatsEst = result.findColumnStatistics(a);
Assertions.assertEquals(100, aStatsEst.minValue);
Assertions.assertEquals(200, aStatsEst.maxValue);
+ Assertions.assertEquals(1.0, aStatsEst.selectivity);
Assertions.assertEquals(10, aStatsEst.ndv);
}
@@ -198,7 +198,7 @@ class FilterEstimationTest {
Statistics stat = new Statistics(1000, slotToColumnStat);
FilterEstimation filterEstimation = new FilterEstimation();
Statistics expected = filterEstimation.estimate(or, stat);
- Assertions.assertEquals(51.9, expected.getRowCount(), 0.1);
+ Assertions.assertEquals(51, expected.getRowCount(), 0.1);
}
// a > 500 and b < 100 or a > c
@@ -418,9 +418,7 @@ class FilterEstimationTest {
.setAvgSizeByte(4)
.setNumNulls(0)
.setMinValue(1)
- .setMinExpr(new IntLiteral(1))
- .setMaxValue(10)
- .setMaxExpr(new IntLiteral(10));
+ .setMaxValue(10);
slotToColumnStat.put(a, builder.build());
Statistics stat = new Statistics(1000, slotToColumnStat);
FilterEstimation filterEstimation = new FilterEstimation();
@@ -469,19 +467,22 @@ class FilterEstimationTest {
.setAvgSizeByte(4)
.setNumNulls(0)
.setMinValue(1000)
- .setMaxValue(10000);
+ .setMaxValue(10000)
+ .setSelectivity(1.0);
ColumnStatisticBuilder builderB = new ColumnStatisticBuilder()
.setNdv(100)
.setAvgSizeByte(4)
.setNumNulls(0)
.setMinValue(0)
- .setMaxValue(500);
+ .setMaxValue(500)
+ .setSelectivity(1.0);
ColumnStatisticBuilder builderC = new ColumnStatisticBuilder()
.setNdv(100)
.setAvgSizeByte(4)
.setNumNulls(0)
.setMinValue(0)
- .setMaxValue(200);
+ .setMaxValue(200)
+ .setSelectivity(1.0);
slotToColumnStat.put(a, builderA.build());
slotToColumnStat.put(b, builderB.build());
slotToColumnStat.put(c, builderC.build());
@@ -503,7 +504,7 @@ class FilterEstimationTest {
* filter range has intersection with (c.min, c.max)
* rows = 100
* a primary key, a.ndv reduced by 1/4
- * b normal field, b.ndv=20
+ * b normal field, b.ndv=20 =>
* c.ndv = 10/40 * c.ndv
*/
@Test
@@ -523,19 +524,22 @@ class FilterEstimationTest {
.setAvgSizeByte(4)
.setNumNulls(0)
.setMinValue(0)
- .setMaxValue(100);
+ .setMaxValue(100)
+ .setSelectivity(1.0);
ColumnStatisticBuilder builderB = new ColumnStatisticBuilder()
.setNdv(20)
.setAvgSizeByte(4)
.setNumNulls(0)
.setMinValue(0)
- .setMaxValue(500);
+ .setMaxValue(500)
+ .setSelectivity(1.0);
ColumnStatisticBuilder builderC = new ColumnStatisticBuilder()
.setNdv(40)
.setAvgSizeByte(4)
.setNumNulls(0)
.setMinValue(0)
- .setMaxValue(40);
+ .setMaxValue(40)
+ .setSelectivity(1.0);
slotToColumnStat.put(a, builderA.build());
slotToColumnStat.put(b, builderB.build());
slotToColumnStat.put(c, builderC.build());
@@ -550,21 +554,25 @@ class FilterEstimationTest {
Assertions.assertEquals(100, statsA.maxValue);
ColumnStatistic statsB = estimated.findColumnStatistics(b);
- Assertions.assertEquals(20, statsB.ndv, 0.1);
+ Assertions.assertEquals(15.6, statsB.ndv, 0.1);
Assertions.assertEquals(0, statsB.minValue);
Assertions.assertEquals(500, statsB.maxValue);
+ Assertions.assertEquals(1.0, statsB.selectivity);
ColumnStatistic statsC = estimated.findColumnStatistics(c);
Assertions.assertEquals(10, statsC.ndv);
Assertions.assertEquals(10, statsC.minValue);
Assertions.assertEquals(20, statsC.maxValue);
+ Assertions.assertEquals(1.0, statsC.selectivity);
}
/**
* test filter estimation, c > 300, where 300 is out of c's range (0,200)
* after filter
+ * c.selectivity=a.selectivity=b.selectivity = 0
* c.ndv=a.ndv=b.ndv=0
+ * a.ndv = b.ndv = 0
*/
@Test
@@ -579,23 +587,23 @@ class FilterEstimationTest {
.setNdv(1000)
.setAvgSizeByte(4)
.setNumNulls(0)
- .setMinValue(1000)
- .setMaxValue(10000)
- .setCount(1000);
+ .setMinValue(10000)
+ .setMaxValue(1000)
+ .setSelectivity(1.0);
ColumnStatisticBuilder builderB = new ColumnStatisticBuilder()
.setNdv(100)
.setAvgSizeByte(4)
.setNumNulls(0)
.setMinValue(0)
.setMaxValue(500)
- .setCount(1000);
+ .setSelectivity(1.0);
ColumnStatisticBuilder builderC = new ColumnStatisticBuilder()
.setNdv(100)
.setAvgSizeByte(4)
.setNumNulls(0)
.setMinValue(0)
.setMaxValue(200)
- .setCount(1000);
+ .setSelectivity(1.0);
slotToColumnStat.put(a, builderA.build());
slotToColumnStat.put(b, builderB.build());
slotToColumnStat.put(c, builderC.build());
@@ -608,8 +616,8 @@ class FilterEstimationTest {
Assertions.assertEquals(0, statsB.ndv);
ColumnStatistic statsC = estimated.findColumnStatistics(c);
Assertions.assertEquals(0, statsC.ndv);
- Assertions.assertTrue(Double.isInfinite(statsC.minValue));
- Assertions.assertTrue(Double.isInfinite(statsC.maxValue));
+ Assertions.assertTrue(Double.isNaN(statsC.minValue));
+ Assertions.assertTrue(Double.isNaN(statsC.maxValue));
}
/**
@@ -652,19 +660,22 @@ class FilterEstimationTest {
.setAvgSizeByte(4)
.setNumNulls(0)
.setMinValue(0)
- .setMaxValue(100);
+ .setMaxValue(100)
+ .setSelectivity(1.0);
ColumnStatisticBuilder builderB = new ColumnStatisticBuilder()
.setNdv(20)
.setAvgSizeByte(4)
.setNumNulls(0)
.setMinValue(0)
- .setMaxValue(500);
+ .setMaxValue(500)
+ .setSelectivity(1.0);
ColumnStatisticBuilder builderC = new ColumnStatisticBuilder()
.setNdv(40)
.setAvgSizeByte(4)
.setNumNulls(0)
.setMinValue(0)
- .setMaxValue(40);
+ .setMaxValue(40)
+ .setSelectivity(1.0);
slotToColumnStat.put(a, builderA.build());
slotToColumnStat.put(b, builderB.build());
slotToColumnStat.put(c, builderC.build());
@@ -679,7 +690,7 @@ class FilterEstimationTest {
Assertions.assertEquals(5, statsA.ndv, 0.1);
Assertions.assertEquals(0, statsA.minValue);
Assertions.assertEquals(100, statsA.maxValue);
- Assertions.assertEquals(5, statsB.ndv, 0.1);
+ Assertions.assertEquals(4.5, statsB.ndv, 0.1);
Assertions.assertEquals(0, statsB.minValue);
Assertions.assertEquals(500, statsB.maxValue);
Assertions.assertEquals(2, statsC.ndv);
@@ -721,6 +732,7 @@ class FilterEstimationTest {
.setNumNulls(0)
.setMinValue(0)
.setMaxValue(100)
+ .setSelectivity(1.0)
.setCount(100);
ColumnStatisticBuilder builderB = new ColumnStatisticBuilder()
.setCount(100)
@@ -728,14 +740,16 @@ class FilterEstimationTest {
.setAvgSizeByte(4)
.setNumNulls(0)
.setMinValue(0)
- .setMaxValue(500);
+ .setMaxValue(500)
+ .setSelectivity(1.0);
ColumnStatisticBuilder builderC = new ColumnStatisticBuilder()
.setCount(100)
.setNdv(40)
.setAvgSizeByte(4)
.setNumNulls(0)
.setMinValue(0)
... 1103 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org