You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2022/12/09 13:12:43 UTC
[doris] branch master updated: [fix](statistics) Fix NPE when update analysis job status #14892
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f5aa5c1d01 [fix](statistics) Fix NPE when update analysis job status #14892
f5aa5c1d01 is described below
commit f5aa5c1d0130444319fea6b1f84e17cb14c4089f
Author: Kikyou1997 <33...@users.noreply.github.com>
AuthorDate: Fri Dec 9 21:12:36 2022 +0800
[fix](statistics) Fix NPE when update analysis job status #14892
---
.../doris/catalog/InternalSchemaInitializer.java | 1 +
.../apache/doris/statistics/AnalysisManager.java | 4 +++-
.../doris/statistics/AnalysisTaskScheduler.java | 7 -------
.../apache/doris/statistics/BaseAnalysisTask.java | 9 ++++++---
.../apache/doris/statistics/ColumnStatistic.java | 15 ++++++++++++---
.../apache/doris/statistics/HiveAnalysisTask.java | 4 ++--
.../doris/statistics/IcebergAnalysisTask.java | 2 +-
.../org/apache/doris/statistics/MVAnalysisTask.java | 6 ++++--
.../apache/doris/statistics/OlapAnalysisTask.java | 3 ++-
.../apache/doris/statistics/StatisticsCache.java | 18 +++++++++++-------
.../apache/doris/statistics/StatisticsCacheKey.java | 10 ++++++++--
.../doris/statistics/StatisticsCacheLoader.java | 3 ++-
.../doris/statistics/StatisticsRepository.java | 21 +++++++++++----------
.../java/org/apache/doris/statistics/CacheTest.java | 4 +++-
.../apache/doris/statistics/MVStatisticsTest.java | 2 +-
15 files changed, 67 insertions(+), 42 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java
index a0c92349c7..7e96b5943f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java
@@ -105,6 +105,7 @@ public class InternalSchemaInitializer extends Thread {
columnDefs.add(new ColumnDef("catalog_id", TypeDef.createVarchar(StatisticConstants.MAX_NAME_LEN)));
columnDefs.add(new ColumnDef("db_id", TypeDef.createVarchar(StatisticConstants.MAX_NAME_LEN)));
columnDefs.add(new ColumnDef("tbl_id", TypeDef.createVarchar(StatisticConstants.MAX_NAME_LEN)));
+ columnDefs.add(new ColumnDef("idx_id", TypeDef.createVarchar(StatisticConstants.MAX_NAME_LEN)));
columnDefs.add(new ColumnDef("col_id", TypeDef.createVarchar(StatisticConstants.MAX_NAME_LEN)));
ColumnDef partId = new ColumnDef("part_id", TypeDef.createVarchar(StatisticConstants.MAX_NAME_LEN));
partId.setAllowNull(true);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index 3833acc20f..cbfd217158 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -80,6 +80,7 @@ public class AnalysisManager {
.setTaskId(taskId).setCatalogName(catalogName).setDbName(db)
.setTblName(tbl.getTbl()).setColName(colName).setJobType(JobType.MANUAL)
.setAnalysisMethod(AnalysisMethod.FULL).setAnalysisType(AnalysisType.COLUMN)
+ .setState(AnalysisState.PENDING)
.setScheduleType(ScheduleType.ONCE).build();
try {
StatisticsRepository.createAnalysisTask(analysisTaskInfo);
@@ -132,7 +133,8 @@ public class AnalysisManager {
} finally {
info.state = jobState;
if (analysisJobIdToTaskMap.get(info.jobId).values()
- .stream().allMatch(i -> i.state != AnalysisState.PENDING && i.state != AnalysisState.RUNNING)) {
+ .stream().allMatch(i -> i.state != null
+ && i.state != AnalysisState.PENDING && i.state != AnalysisState.RUNNING)) {
analysisJobIdToTaskMap.remove(info.jobId);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskScheduler.java
index 71f23fe955..3e2a7bf1c5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskScheduler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskScheduler.java
@@ -30,7 +30,6 @@ import org.apache.log4j.Logger;
import java.util.Comparator;
import java.util.HashSet;
import java.util.LinkedList;
-import java.util.List;
import java.util.PriorityQueue;
import java.util.Queue;
import java.util.Set;
@@ -48,12 +47,6 @@ public class AnalysisTaskScheduler {
private final Set<BaseAnalysisTask> manualJobSet = new HashSet<>();
- public synchronized void scheduleJobs(List<AnalysisTaskInfo> analysisJobInfos) {
- for (AnalysisTaskInfo job : analysisJobInfos) {
- schedule(job);
- }
- }
-
public synchronized void schedule(AnalysisTaskInfo analysisJobInfo) {
CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(analysisJobInfo.catalogName);
Preconditions.checkArgument(catalog != null);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
index 79b0ec8c16..81c4601647 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
@@ -32,10 +32,11 @@ public abstract class BaseAnalysisTask {
protected static final String INSERT_PART_STATISTICS = "INSERT INTO "
+ "${internalDB}.${columnStatTbl}"
+ " SELECT "
- + "CONCAT(${tblId}, '-', '${colId}', '-', ${partId}) AS id, "
+ + "CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}', '-', ${partId}) AS id, "
+ "${catalogId} AS catalog_id, "
+ "${dbId} AS db_id, "
+ "${tblId} AS tbl_id, "
+ + "${idxId} AS idx_id, "
+ "'${colId}' AS col_id, "
+ "${partId} AS part_id, "
+ "COUNT(1) AS row_count, "
@@ -48,13 +49,14 @@ public abstract class BaseAnalysisTask {
protected static final String INSERT_COL_STATISTICS = "INSERT INTO "
+ "${internalDB}.${columnStatTbl}"
- + " SELECT id, catalog_id, db_id, tbl_id, col_id, part_id, row_count, "
+ + " SELECT id, catalog_id, db_id, tbl_id, idx_id, col_id, part_id, row_count, "
+ " ndv, null_count, min, max, data_size, update_time\n"
+ " FROM \n"
- + " (SELECT CONCAT(${tblId}, '-', '${colId}') AS id, "
+ + " (SELECT CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}') AS id, "
+ " ${catalogId} AS catalog_id, "
+ " ${dbId} AS db_id, "
+ " ${tblId} AS tbl_id, "
+ + " ${idxId} AS idx_id, "
+ " '${colId}' AS col_id, "
+ " NULL AS part_id, "
+ " SUM(count) AS row_count, \n"
@@ -67,6 +69,7 @@ public abstract class BaseAnalysisTask {
+ " WHERE ${internalDB}.${columnStatTbl}.db_id = '${dbId}' AND "
+ " ${internalDB}.${columnStatTbl}.tbl_id='${tblId}' AND "
+ " ${internalDB}.${columnStatTbl}.col_id='${colId}' AND "
+ + " ${internalDB}.${columnStatTbl}.idx_id='${idxId}' AND "
+ " ${internalDB}.${columnStatTbl}.part_id IS NOT NULL"
+ " ) t1, \n";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
index a5aee62c8e..4865767496 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
@@ -21,6 +21,7 @@ import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.Type;
import org.apache.doris.datasource.CatalogIf;
@@ -120,12 +121,14 @@ public class ColumnStatistic {
columnStatisticBuilder.setAvgSizeByte(columnStatisticBuilder.getDataSize()
/ columnStatisticBuilder.getCount());
long catalogId = Long.parseLong(resultRow.getColumnValue("catalog_id"));
+ long idxId = Long.parseLong(resultRow.getColumnValue("idx_id"));
long dbID = Long.parseLong(resultRow.getColumnValue("db_id"));
long tblId = Long.parseLong(resultRow.getColumnValue("tbl_id"));
String colName = resultRow.getColumnValue("col_id");
- Column col = findColumn(catalogId, dbID, tblId, colName);
+ Column col = findColumn(catalogId, dbID, tblId, idxId, colName);
if (col == null) {
- LOG.warn("Failed to deserialize column statistics, column:{}.{}.{}.{} not exists",
+ LOG.warn("Failed to deserialize column statistics, ctlId: {} dbId: {}"
+ + "tblId: {} column: {} not exists",
catalogId, dbID, tblId, colName);
return ColumnStatistic.DEFAULT;
}
@@ -175,7 +178,7 @@ public class ColumnStatistic {
return Math.max(this.minValue, other.minValue) <= Math.min(this.maxValue, other.maxValue);
}
- public static Column findColumn(long catalogId, long dbId, long tblId, String columnName) {
+ public static Column findColumn(long catalogId, long dbId, long tblId, long idxId, String columnName) {
CatalogIf<DatabaseIf<TableIf>> catalogIf = Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogId);
if (catalogIf == null) {
return null;
@@ -188,6 +191,12 @@ public class ColumnStatistic {
if (tblIf == null) {
return null;
}
+ if (idxId != -1) {
+ if (tblIf instanceof OlapTable) {
+ OlapTable olapTable = (OlapTable) tblIf;
+ return olapTable.getIndexIdToMeta().get(idxId).getColumnByName(columnName);
+ }
+ }
return tblIf.getColumn(columnName);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HiveAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HiveAnalysisTask.java
index 7bba21c692..4ad00bdc75 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HiveAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HiveAnalysisTask.java
@@ -57,12 +57,12 @@ public class HiveAnalysisTask extends HMSAnalysisTask {
private static final String ANALYZE_PARTITION_SQL_TEMPLATE = "INSERT INTO "
+ "${internalDB}.${columnStatTbl}"
- + " values ('${id}','${catalogId}', '${dbId}', '${tblId}', '${colId}', '${partId}', "
+ + " values ('${id}','${catalogId}', '${dbId}', '${tblId}', '-1', '${colId}', '${partId}', "
+ "${numRows}, ${ndv}, ${nulls}, '${min}', '${max}', ${dataSize}, '${update_time}')";
private static final String ANALYZE_TABLE_SQL_TEMPLATE = "INSERT INTO "
+ "${internalDB}.${columnStatTbl}"
- + " values ('${id}','${catalogId}', '${dbId}', '${tblId}', '${colId}', NULL, "
+ + " values ('${id}','${catalogId}', '${dbId}', '${tblId}', 'NULL', '${colId}', NULL, "
+ "${numRows}, ${ndv}, ${nulls}, '${min}', '${max}', ${dataSize}, '${update_time}')";
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/IcebergAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/IcebergAnalysisTask.java
index b51fa4eb53..103aceb714 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/IcebergAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/IcebergAnalysisTask.java
@@ -50,7 +50,7 @@ public class IcebergAnalysisTask extends HMSAnalysisTask {
private static final String INSERT_TABLE_SQL_TEMPLATE = "INSERT INTO "
+ "${internalDB}.${columnStatTbl}"
- + " values ('${id}','${catalogId}', '${dbId}', '${tblId}', '${colId}', NULL, "
+ + " values ('${id}','${catalogId}', '${dbId}', '${tblId}', '-1', '${colId}', NULL, "
+ "${numRows}, 0, ${nulls}, '0', '0', ${dataSize}, '${update_time}')";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java
index 70f4d0c5f3..5c66fd062b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java
@@ -107,7 +107,8 @@ public class MVAnalysisTask extends BaseAnalysisTask {
params.put("columnStatTbl", StatisticConstants.STATISTIC_TBL_NAME);
params.put("catalogId", String.valueOf(catalog.getId()));
params.put("dbId", String.valueOf(db.getId()));
- params.put("tblId", String.valueOf(meta.getIndexId()));
+ params.put("tblId", String.valueOf(tbl.getId()));
+ params.put("idxId", String.valueOf(meta.getIndexId()));
String colName = column.getName();
params.put("colId", colName);
long partId = part.getId();
@@ -122,7 +123,8 @@ public class MVAnalysisTask extends BaseAnalysisTask {
params.remove("partId");
params.put("type", column.getType().toString());
StatisticsUtil.execUpdate(ANALYZE_MV_COL, params);
- Env.getCurrentEnv().getStatisticsCache().refreshSync(meta.getIndexId(), column.getName());
+ Env.getCurrentEnv().getStatisticsCache()
+ .refreshSync(meta.getIndexId(), meta.getIndexId(), column.getName());
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
index 23ea5ea6a6..97cd92c29c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
@@ -62,6 +62,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
params.put("catalogId", String.valueOf(catalog.getId()));
params.put("dbId", String.valueOf(db.getId()));
params.put("tblId", String.valueOf(tbl.getId()));
+ params.put("idxId", "-1");
params.put("colId", String.valueOf(info.colName));
params.put("dataSizeFunction", getDataSizeFunction(col));
params.put("dbName", info.dbName);
@@ -90,7 +91,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
String sql = stringSubstitutor.replace(ANALYZE_COLUMN_SQL_TEMPLATE);
execSQL(sql);
- Env.getCurrentEnv().getStatisticsCache().refreshSync(tbl.getId(), col.getName());
+ Env.getCurrentEnv().getStatisticsCache().refreshSync(tbl.getId(), -1, col.getName());
}
@VisibleForTesting
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
index fac8448f3d..b92873460c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
@@ -38,10 +38,14 @@ public class StatisticsCache {
.buildAsync(new StatisticsCacheLoader());
public ColumnStatistic getColumnStatistics(long tblId, String colName) {
+ return getColumnStatistics(tblId, -1, colName);
+ }
+
+ public ColumnStatistic getColumnStatistics(long tblId, long idxId, String colName) {
if (ConnectContext.get().getSessionVariable().internalSession) {
return ColumnStatistic.DEFAULT;
}
- StatisticsCacheKey k = new StatisticsCacheKey(tblId, colName);
+ StatisticsCacheKey k = new StatisticsCacheKey(tblId, idxId, colName);
try {
CompletableFuture<ColumnStatistic> f = cache.get(k);
if (f.isDone()) {
@@ -55,16 +59,16 @@ public class StatisticsCache {
}
// TODO: finish this method.
- public void eraseExpiredCache(long tblId, String colName) {
- cache.synchronous().invalidate(new StatisticsCacheKey(tblId, colName));
+ public void eraseExpiredCache(long tblId, long idxId, String colName) {
+ cache.synchronous().invalidate(new StatisticsCacheKey(tblId, idxId, colName));
}
- public void updateCache(long tblId, String colName, ColumnStatistic statistic) {
+ public void updateCache(long tblId, long idxId, String colName, ColumnStatistic statistic) {
- cache.synchronous().put(new StatisticsCacheKey(tblId, colName), statistic);
+ cache.synchronous().put(new StatisticsCacheKey(tblId, idxId, colName), statistic);
}
- public void refreshSync(long tblId, String colName) {
- cache.synchronous().refresh(new StatisticsCacheKey(tblId, colName));
+ public void refreshSync(long tblId, long idxId, String colName) {
+ cache.synchronous().refresh(new StatisticsCacheKey(tblId, idxId, colName));
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheKey.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheKey.java
index 1aaa98d154..9b48876bc2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheKey.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheKey.java
@@ -25,16 +25,22 @@ public class StatisticsCacheKey {
* May be index id either, since they are natively same in the code.
*/
public final long tableId;
+ public final long idxId;
public final String colName;
public StatisticsCacheKey(long tableId, String colName) {
+ this(tableId, -1, colName);
+ }
+
+ public StatisticsCacheKey(long tableId, long idxId, String colName) {
this.tableId = tableId;
+ this.idxId = idxId;
this.colName = colName;
}
@Override
public int hashCode() {
- return Objects.hash(tableId, colName);
+ return Objects.hash(tableId, idxId, colName);
}
@Override
@@ -46,6 +52,6 @@ public class StatisticsCacheKey {
return false;
}
StatisticsCacheKey k = (StatisticsCacheKey) obj;
- return this.tableId == k.tableId && this.colName.equals(k.colName);
+ return this.tableId == k.tableId && this.idxId == k.idxId && this.colName.equals(k.colName);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheLoader.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheLoader.java
index d27f5893b5..dea76b6201 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheLoader.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheLoader.java
@@ -41,7 +41,7 @@ public class StatisticsCacheLoader implements AsyncCacheLoader<StatisticsCacheKe
private static final String QUERY_COLUMN_STATISTICS = "SELECT * FROM " + FeConstants.INTERNAL_DB_NAME
+ "." + StatisticConstants.STATISTIC_TBL_NAME + " WHERE "
- + "id = CONCAT('${tblId}', '-', '${colId}')";
+ + "id = CONCAT('${tblId}', '-', ${idxId}, '-', '${colId}')";
private static int CUR_RUNNING_LOAD = 0;
@@ -64,6 +64,7 @@ public class StatisticsCacheLoader implements AsyncCacheLoader<StatisticsCacheKe
try {
Map<String, String> params = new HashMap<>();
params.put("tblId", String.valueOf(key.tableId));
+ params.put("idxId", String.valueOf(key.idxId));
params.put("colId", String.valueOf(key.colName));
List<ResultRow> resultBatches =
StatisticsUtil.execStatisticQuery(new StringSubstitutor(params)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
index 715ec14464..78693bbaef 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
@@ -66,15 +66,15 @@ public class StatisticsRepository {
+ FULL_QUALIFIED_COLUMN_STATISTICS_NAME
+ " WHERE `id` IN (${idList})";
- private static final String PERSIST_ANALYSIS_JOB_SQL_TEMPLATE = "INSERT INTO "
+ private static final String PERSIST_ANALYSIS_TASK_SQL_TEMPLATE = "INSERT INTO "
+ FULL_QUALIFIED_ANALYSIS_JOB_TABLE_NAME + " VALUES(${jobId}, ${taskId}, '${catalogName}', '${dbName}',"
- + "'${tblName}','${colName}', ,'${indexId}','${jobType}', '${analysisType}', "
+ + "'${tblName}','${colName}', '${indexId}','${jobType}', '${analysisType}', "
+ "'${message}', '${lastExecTimeInMs}',"
+ "'${state}', '${scheduleType}')";
private static final String INSERT_INTO_COLUMN_STATISTICS = "INSERT INTO "
- + FULL_QUALIFIED_COLUMN_STATISTICS_NAME + " VALUES('${id}', ${catalogId}, ${dbId}, ${tblId}, '${colId}',"
- + "${partId}, ${count}, ${ndv}, ${nullCount}, '${min}', '${max}', ${dataSize}, NOW())";
+ + FULL_QUALIFIED_COLUMN_STATISTICS_NAME + " VALUES('${id}', ${catalogId}, ${dbId}, ${tblId}, '${idxId}',"
+ + "'${colId}', ${partId}, ${count}, ${ndv}, ${nullCount}, '${min}', '${max}', ${dataSize}, NOW())";
public static ColumnStatistic queryColumnStatisticsByName(long tableId, String colName) {
ResultRow resultRow = queryColumnStatisticById(tableId, colName);
@@ -102,7 +102,7 @@ public class StatisticsRepository {
public static ResultRow queryColumnStatisticById(long tblId, String colName) {
Map<String, String> map = new HashMap<>();
- String id = constructId(tblId, colName);
+ String id = constructId(tblId, -1, colName);
map.put("id", id);
List<ResultRow> rows = StatisticsUtil.executeQuery(FETCH_COLUMN_STATISTIC_TEMPLATE, map);
int size = rows.size();
@@ -115,7 +115,7 @@ public class StatisticsRepository {
public static List<ResultRow> queryPartitionStatistics(long tblId, String colName, Set<Long> partIds) {
StringJoiner sj = new StringJoiner(",");
for (Long partId : partIds) {
- sj.add("'" + constructId(tblId, colName, partId) + "'");
+ sj.add("'" + constructId(tblId, -1, colName, partId) + "'");
}
Map<String, String> params = new HashMap<>();
params.put("idList", sj.toString());
@@ -139,7 +139,7 @@ public class StatisticsRepository {
params.put("dbName", analysisTaskInfo.dbName);
params.put("tblName", analysisTaskInfo.tblName);
params.put("colName", analysisTaskInfo.colName);
- params.put("indexId", String.valueOf(analysisTaskInfo.indexId));
+ params.put("indexId", analysisTaskInfo.indexId == null ? "-1" : String.valueOf(analysisTaskInfo.indexId));
params.put("jobType", analysisTaskInfo.jobType.toString());
params.put("analysisType", analysisTaskInfo.analysisMethod.toString());
params.put("message", "");
@@ -147,7 +147,7 @@ public class StatisticsRepository {
params.put("state", AnalysisState.PENDING.toString());
params.put("scheduleType", analysisTaskInfo.scheduleType.toString());
StatisticsUtil.execUpdate(
- new StringSubstitutor(params).replace(PERSIST_ANALYSIS_JOB_SQL_TEMPLATE));
+ new StringSubstitutor(params).replace(PERSIST_ANALYSIS_TASK_SQL_TEMPLATE));
}
public static void alterColumnStatistics(AlterColumnStatsStmt alterColumnStatsStmt) throws Exception {
@@ -184,9 +184,10 @@ public class StatisticsRepository {
}
ColumnStatistic columnStatistic = builder.build();
Map<String, String> params = new HashMap<>();
- params.put("id", constructId(objects.table.getId(), colName));
+ params.put("id", constructId(objects.table.getId(), -1, colName));
params.put("catalogId", String.valueOf(objects.catalog.getId()));
params.put("dbId", String.valueOf(objects.db.getId()));
+ params.put("idxId", "-1");
params.put("tblId", String.valueOf(objects.table.getId()));
params.put("colId", String.valueOf(colName));
params.put("partId", "NULL");
@@ -197,6 +198,6 @@ public class StatisticsRepository {
params.put("max", max == null ? "NULL" : max);
params.put("dataSize", String.valueOf(columnStatistic.dataSize));
StatisticsUtil.execUpdate(INSERT_INTO_COLUMN_STATISTICS, params);
- Env.getCurrentEnv().getStatisticsCache().updateCache(objects.table.getId(), colName, builder.build());
+ Env.getCurrentEnv().getStatisticsCache().updateCache(objects.table.getId(), -1, colName, builder.build());
}
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java
index 2d41c68599..c61b692870 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java
@@ -66,7 +66,7 @@ public class CacheTest extends TestWithFeService {
new MockUp<ColumnStatistic>() {
@Mock
- public Column findColumn(long catalogId, long dbId, long tblId, String columnName) {
+ public Column findColumn(long catalogId, long dbId, long tblId, long idxId, String columnName) {
return new Column("abc", PrimitiveType.BIGINT);
}
};
@@ -86,6 +86,7 @@ public class CacheTest extends TestWithFeService {
colNames.add("data_size_in_bytes");
colNames.add("catalog_id");
colNames.add("db_id");
+ colNames.add("idx_id");
colNames.add("tbl_id");
colNames.add("col_id");
colNames.add("min");
@@ -107,6 +108,7 @@ public class CacheTest extends TestWithFeService {
values.add("3");
values.add("4");
values.add("5");
+ values.add("-1");
values.add("6");
values.add("7");
values.add("8");
diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/MVStatisticsTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/MVStatisticsTest.java
index 7188342b83..7ed261c831 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/statistics/MVStatisticsTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/MVStatisticsTest.java
@@ -54,7 +54,7 @@ public class MVStatisticsTest extends TestWithFeService {
public void testCreate() throws Exception {
new Expectations() {
{
- statisticsCache.refreshSync(anyLong, anyString);
+ statisticsCache.refreshSync(anyLong, anyLong, anyString);
times = 5;
}
};
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org