You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/06/15 08:29:01 UTC
[incubator-doris] branch master updated: [fix] Fix disk used pct only consider the data that used by Doris (#9705)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 49f4437396 [fix] Fix disk used pct only consider the data that used by Doris (#9705)
49f4437396 is described below
commit 49f443739629dd0623d42e0aab80880de7f7d74c
Author: ccoffline <45...@users.noreply.github.com>
AuthorDate: Wed Jun 15 16:28:56 2022 +0800
[fix] Fix disk used pct only consider the data that used by Doris (#9705)
---
.../src/main/java/org/apache/doris/catalog/DiskInfo.java | 12 +++++++-----
.../java/org/apache/doris/clone/BackendLoadStatistic.java | 2 +-
.../java/org/apache/doris/clone/DiskRebalanceTest.java | 15 ++++++++++++---
.../java/org/apache/doris/clone/RebalancerTestUtil.java | 1 +
4 files changed, 21 insertions(+), 9 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DiskInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DiskInfo.java
index 0f8b1c5454..a96b83869e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DiskInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DiskInfo.java
@@ -89,6 +89,10 @@ public class DiskInfo implements Writable {
this.dataUsedCapacityB = dataUsedCapacityB;
}
+ public long getDiskUsedCapacityB() {
+ return totalCapacityB - diskAvailableCapacityB;
+ }
+
public long getAvailableCapacityB() {
return diskAvailableCapacityB;
}
@@ -98,7 +102,7 @@ public class DiskInfo implements Writable {
}
public double getUsedPct() {
- return (totalCapacityB - diskAvailableCapacityB) / (double) (totalCapacityB <= 0 ? 1 : totalCapacityB);
+ return this.getDiskUsedCapacityB() / (double) (totalCapacityB <= 0 ? 1 : totalCapacityB);
}
public DiskState getState() {
@@ -148,12 +152,10 @@ public class DiskInfo implements Writable {
floodStage, diskAvailableCapacityB, totalCapacityB);
if (floodStage) {
return diskAvailableCapacityB < Config.storage_flood_stage_left_capacity_bytes
- && (double) (totalCapacityB - diskAvailableCapacityB) / totalCapacityB
- > (Config.storage_flood_stage_usage_percent / 100.0);
+ && this.getUsedPct() > (Config.storage_flood_stage_usage_percent / 100.0);
} else {
return diskAvailableCapacityB < Config.storage_min_left_capacity_bytes
- || (double) (totalCapacityB - diskAvailableCapacityB) / totalCapacityB
- > (Config.storage_high_watermark_usage_percent / 100.0);
+ || this.getUsedPct() > (Config.storage_high_watermark_usage_percent / 100.0);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java b/fe/fe-core/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java
index bcc5edcab9..824137b2f1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java
@@ -187,7 +187,7 @@ public class BackendLoadStatistic {
RootPathLoadStatistic pathStatistic = new RootPathLoadStatistic(beId, diskInfo.getRootPath(),
diskInfo.getPathHash(), diskInfo.getStorageMedium(),
- diskInfo.getTotalCapacityB(), diskInfo.getDataUsedCapacityB(), diskInfo.getState());
+ diskInfo.getTotalCapacityB(), diskInfo.getDiskUsedCapacityB(), diskInfo.getState());
pathStatistics.add(pathStatistic);
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/clone/DiskRebalanceTest.java b/fe/fe-core/src/test/java/org/apache/doris/clone/DiskRebalanceTest.java
index 1fbd3a8dd4..b4bc9ce249 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/clone/DiskRebalanceTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/clone/DiskRebalanceTest.java
@@ -190,9 +190,12 @@ public class DiskRebalanceTest {
@Test
public void testDiskRebalancerWithDiffUsageDisk() {
// init system
- systemInfoService.addBackend(RebalancerTestUtil.createBackend(10001L, 2048, Lists.newArrayList(1024L), 1));
- systemInfoService.addBackend(RebalancerTestUtil.createBackend(10002L, 2048, Lists.newArrayList(1024L, 512L), 2));
- systemInfoService.addBackend(RebalancerTestUtil.createBackend(10003L, 2048, Lists.newArrayList(1024L, 512L, 513L), 3));
+ systemInfoService.addBackend(RebalancerTestUtil.createBackend(10001L, 2048,
+ Lists.newArrayList(1024L), 1));
+ systemInfoService.addBackend(RebalancerTestUtil.createBackend(10002L, 2048,
+ Lists.newArrayList(1024L, 512L), 2));
+ systemInfoService.addBackend(RebalancerTestUtil.createBackend(10003L, 2048,
+ Lists.newArrayList(1024L, 512L, 1024L), 3));
olapTable = new OlapTable(2, "fake table", new ArrayList<>(), KeysType.DUP_KEYS,
new RangePartitionInfo(), new HashDistributionInfo());
@@ -221,6 +224,12 @@ public class DiskRebalanceTest {
Rebalancer rebalancer = new DiskRebalancer(Catalog.getCurrentSystemInfo(), Catalog.getCurrentInvertedIndex());
generateStatisticMap();
rebalancer.updateLoadStatistic(statisticMap);
+ for (Table.Cell<String, Tag, ClusterLoadStatistic> s : statisticMap.cellSet()) {
+ if (s.getValue() != null) {
+ LOG.info("cluster = {}, tag = {}, statistic = {}",
+ s.getRowKey(), s.getColumnKey(), s.getValue().getBrief());
+ }
+ }
List<TabletSchedCtx> alternativeTablets = rebalancer.selectAlternativeTablets();
// check alternativeTablets;
Assert.assertEquals(2, alternativeTablets.size());
diff --git a/fe/fe-core/src/test/java/org/apache/doris/clone/RebalancerTestUtil.java b/fe/fe-core/src/test/java/org/apache/doris/clone/RebalancerTestUtil.java
index 8f6f9f6f58..2889742187 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/clone/RebalancerTestUtil.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/clone/RebalancerTestUtil.java
@@ -54,6 +54,7 @@ public class RebalancerTestUtil {
diskInfo.setPathHash(id + i);
diskInfo.setTotalCapacityB(totalCap);
diskInfo.setDataUsedCapacityB(usedCaps.get(i));
+ diskInfo.setAvailableCapacityB(totalCap - usedCaps.get(i));
disks.put(diskInfo.getRootPath(), diskInfo);
}
be.setDisks(ImmutableMap.copyOf(disks));
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org