You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/01/31 15:54:45 UTC
(doris) 33/37: [fix](statistics)Fix stats cleaner delete partition stats bug (#30648)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9310dbf3fd04941d77838e8c65ae234fccd4c9ee
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Wed Jan 31 20:50:33 2024 +0800
[fix](statistics)Fix stats cleaner delete partition stats bug (#30648)
When a partition in OlapTable is removed, we should use partition id to delete the related stats record in column_statistics. Before, it was using id, which may cause delete useful stats of other partition.
---
.../java/org/apache/doris/statistics/StatisticsCleaner.java | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
index 1af219baa6f..3a142351a28 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
@@ -169,12 +169,15 @@ public class StatisticsCleaner extends MasterDaemon {
doDelete("idx_id", expiredStats.expiredIdxId.stream()
.map(String::valueOf).collect(Collectors.toList()),
FeConstants.INTERNAL_DB_NAME + "." + tblName, false);
+ doDelete("part_id", expiredStats.expiredPartitionId.stream()
+ .map(String::valueOf).collect(Collectors.toList()),
+ FeConstants.INTERNAL_DB_NAME + "." + tblName, false);
doDelete("id", expiredStats.ids.stream()
.map(String::valueOf).collect(Collectors.toList()),
FeConstants.INTERNAL_DB_NAME + "." + tblName, false);
}
- private void doDelete(String/*col name*/ colName, List<String> pred, String tblName, boolean taskOnly) {
+ private void doDelete(String colName, List<String> pred, String tblName, boolean taskOnly) {
String deleteTemplate = "DELETE FROM " + tblName + " WHERE ${left} IN (${right})";
if (CollectionUtils.isEmpty(pred)) {
return;
@@ -241,7 +244,7 @@ public class StatisticsCleaner extends MasterDaemon {
continue;
}
if (!olapTable.getPartitionIds().contains(Long.parseLong(partId))) {
- expiredStats.ids.add(id);
+ expiredStats.expiredPartitionId.add(Long.parseLong(partId));
}
} catch (Exception e) {
LOG.warn("Error occurred when retrieving expired stats", e);
@@ -256,9 +259,8 @@ public class StatisticsCleaner extends MasterDaemon {
Set<Long> expiredCatalog = new HashSet<>();
Set<Long> expiredDatabase = new HashSet<>();
Set<Long> expiredTable = new HashSet<>();
-
Set<Long> expiredIdxId = new HashSet<>();
-
+ Set<Long> expiredPartitionId = new HashSet<>();
Set<String> ids = new HashSet<>();
public boolean isFull() {
@@ -266,6 +268,7 @@ public class StatisticsCleaner extends MasterDaemon {
|| expiredDatabase.size() >= Config.max_allowed_in_element_num_of_delete
|| expiredTable.size() >= Config.max_allowed_in_element_num_of_delete
|| expiredIdxId.size() >= Config.max_allowed_in_element_num_of_delete
+ || expiredPartitionId.size() >= Config.max_allowed_in_element_num_of_delete
|| ids.size() >= Config.max_allowed_in_element_num_of_delete;
}
@@ -274,6 +277,7 @@ public class StatisticsCleaner extends MasterDaemon {
&& expiredDatabase.isEmpty()
&& expiredTable.isEmpty()
&& expiredIdxId.isEmpty()
+ && expiredPartitionId.isEmpty()
&& ids.size() < Config.max_allowed_in_element_num_of_delete / 10;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org