You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by sa...@apache.org on 2026/03/31 02:39:50 UTC
(ozone) branch master updated: HDDS-14866. Enhance DiskBalancer Report to show individual volume's density (#9969)
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 1b83e169815 HDDS-14866. Enhance DiskBalancer Report to show individual volume's density (#9969)
1b83e169815 is described below
commit 1b83e1698159bebfb0fb5447f2f6ac37e326cdd5
Author: Gargi Jaiswal <13...@users.noreply.github.com>
AuthorDate: Tue Mar 31 08:09:44 2026 +0530
HDDS-14866. Enhance DiskBalancer Report to show individual volume's density (#9969)
---
.../container/diskbalancer/DiskBalancerInfo.java | 26 +++-
.../diskbalancer/DiskBalancerProtocolServer.java | 2 +
.../diskbalancer/DiskBalancerService.java | 39 +++++-
.../TestDiskBalancerProtocolServer.java | 61 ++++++++-
.../interface-client/src/main/proto/hdds.proto | 12 ++
.../cli/datanode/DiskBalancerReportSubcommand.java | 144 ++++++++++++++++-----
.../cli/datanode/TestDiskBalancerSubCommands.java | 70 +++++++++-
7 files changed, 312 insertions(+), 42 deletions(-)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerInfo.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerInfo.java
index 15cfadbabdd..4492047844d 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerInfo.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerInfo.java
@@ -17,11 +17,15 @@
package org.apache.hadoop.ozone.container.diskbalancer;
+import java.util.Collections;
+import java.util.List;
import java.util.Objects;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DiskBalancerRunningStatus;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.VolumeReportProto;
/**
- * DiskBalancer's information to persist.
+ * DiskBalancer's information to persist and for report.
+ * Report-only fields (idealUsage, volumeInfo) are NOT persisted to YAML.
*/
public class DiskBalancerInfo {
private DiskBalancerRunningStatus operationalState;
@@ -35,6 +39,10 @@ public class DiskBalancerInfo {
private long bytesToMove;
private long balancedBytes;
private double volumeDataDensity;
+ // Report-only: ideal usage from volume snapshot. NOT persisted.
+ private double idealUsage;
+ // Report-only: per-volume info. NOT persisted.
+ private List<VolumeReportProto> volumeInfo;
public DiskBalancerInfo(DiskBalancerRunningStatus operationalState, double threshold,
long bandwidthInMB, int parallelThread, boolean stopAfterDiskEven) {
@@ -208,6 +216,22 @@ public void setVolumeDataDensity(double volumeDataDensity) {
this.volumeDataDensity = volumeDataDensity;
}
+ public double getIdealUsage() {
+ return idealUsage;
+ }
+
+ public void setIdealUsage(double idealUsage) {
+ this.idealUsage = idealUsage;
+ }
+
+ public List<VolumeReportProto> getVolumeInfo() {
+ return volumeInfo != null ? volumeInfo : Collections.emptyList();
+ }
+
+ public void setVolumeInfo(List<VolumeReportProto> volumeInfo) {
+ this.volumeInfo = volumeInfo;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerProtocolServer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerProtocolServer.java
index b92391dc8b9..4bde1cedc49 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerProtocolServer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerProtocolServer.java
@@ -82,6 +82,8 @@ public DatanodeDiskBalancerInfoProto getDiskBalancerInfo(GetDiskBalancerInfoRequ
.setBytesToMove(info.getBytesToMove())
.setBytesMoved(info.getBalancedBytes())
.setRunningStatus(info.getOperationalState())
+ .setIdealUsage(info.getIdealUsage())
+ .addAllVolumeInfo(info.getVolumeInfo())
.build();
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
index 596cf87fde3..b52a08e2641 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
@@ -31,6 +31,8 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -47,6 +49,7 @@
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DiskBalancerRunningStatus;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.VolumeReportProto;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.hdds.server.ServerUtils;
@@ -685,9 +688,43 @@ public DiskBalancerInfo getDiskBalancerInfo() {
bytesToMove = calculateBytesToMove(volumeUsages);
}
- return new DiskBalancerInfo(operationalState, threshold, bandwidthInMB,
+ DiskBalancerInfo info = new DiskBalancerInfo(operationalState, threshold, bandwidthInMB,
parallelThread, stopAfterDiskEven, version, metrics.getSuccessCount(),
metrics.getFailureCount(), bytesToMove, metrics.getSuccessBytes(), volumeDataDensity);
+ info.setIdealUsage(getIdealUsage(volumeUsages));
+ info.setVolumeInfo(buildVolumeReportProto(volumeUsages));
+ return info;
+ }
+
+ /**
+ * Build a list of VolumeReportProto from a list of VolumeFixedUsage.
+ * VolumeReportProto consists of information like StorageID,
+ * volume utilization and committed bytes to the client.
+ *
+ * @param volumeSet snapshot of VolumeFixedUsage which contains the usage information of each volume
+ * @return a list of VolumeReportProto which will be sent to clients for reporting volume status
+ */
+ public static List<VolumeReportProto> buildVolumeReportProto(List<VolumeFixedUsage> volumeSet) {
+ if (volumeSet == null || volumeSet.isEmpty()) {
+ return Collections.emptyList();
+ }
+
+ List<VolumeReportProto> result = new ArrayList<>();
+ for (VolumeFixedUsage v : volumeSet) {
+ HddsVolume volume = v.getVolume();
+ VolumeReportProto.Builder builder = VolumeReportProto.newBuilder()
+ .setStorageId(volume.getStorageID())
+ .setTotalCapacity(v.getUsage().getCapacity())
+ .setUsedSpace(v.getUsage().getUsedSpace())
+ .setCommittedBytes(volume.getCommittedBytes())
+ .setEffectiveUsedSpace(v.getEffectiveUsed())
+ .setUtilization(v.getUtilization());
+ if (volume.getStorageDir() != null) {
+ builder.setStoragePath(volume.getStorageDir().getPath());
+ }
+ result.add(builder.build());
+ }
+ return result;
}
public long calculateBytesToMove(List<VolumeFixedUsage> inputVolumeSet) {
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerProtocolServer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerProtocolServer.java
index 361f5cce051..e0c6e924217 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerProtocolServer.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerProtocolServer.java
@@ -28,12 +28,14 @@
import static org.mockito.Mockito.when;
import java.io.IOException;
+import java.util.Arrays;
import java.util.UUID;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.DiskBalancerProtocolProtos.GetDiskBalancerInfoRequestProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDiskBalancerInfoProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DiskBalancerConfigurationProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DiskBalancerRunningStatus;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.VolumeReportProto;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerProtocolServer.PrivilegedOperation;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
@@ -51,6 +53,23 @@ class TestDiskBalancerProtocolServer {
private static final int TEST_THREADS = 5;
private static final boolean TEST_STOP_AFTER_DISK_EVEN = true;
private static final double TEST_VOLUME_DENSITY = 15.5;
+ private static final double TEST_IDEAL_USAGE = 0.6;
+ private static final String TEST_STORAGE_ID_1 = "vol1";
+ private static final String TEST_STORAGE_PATH_1 = "/data/hdds1";
+ private static final String TEST_STORAGE_ID_2 = "vol2";
+ private static final String TEST_STORAGE_PATH_2 = "/data/hdds2";
+ private static final double TEST_UTILIZATION_1 = 0.57;
+ private static final double TEST_UTILIZATION_2 = 0.63;
+ private static final long TEST_COMMITTED_BYTES_1 = 10L * 1024 * 1024;
+ private static final long TEST_COMMITTED_BYTES_2 = 25L * 1024 * 1024;
+ private static final long TEST_TOTAL_CAPACITY = 200L * 1024 * 1024;
+ private static final long TEST_USED_SPACE_1 = 40L * 1024 * 1024;
+ private static final long TEST_USED_SPACE_2 = 50L * 1024 * 1024;
+ private static final long TEST_EFFECTIVE_USED_SPACE_1 =
+ TEST_USED_SPACE_1 + TEST_COMMITTED_BYTES_1;
+ private static final long TEST_EFFECTIVE_USED_SPACE_2 =
+ TEST_USED_SPACE_2 + TEST_COMMITTED_BYTES_2;
+ private static final int TEST_VOLUME_INFO_COUNT = 2;
private DatanodeStateMachine datanodeStateMachine;
private DiskBalancerService diskBalancerService;
@@ -80,6 +99,27 @@ void setup() throws IOException {
0L, // balancedBytes
TEST_VOLUME_DENSITY
);
+ diskBalancerInfo.setIdealUsage(TEST_IDEAL_USAGE);
+ diskBalancerInfo.setVolumeInfo(Arrays.asList(
+ VolumeReportProto.newBuilder()
+ .setStorageId(TEST_STORAGE_ID_1)
+ .setStoragePath(TEST_STORAGE_PATH_1)
+ .setUtilization(TEST_UTILIZATION_1)
+ .setCommittedBytes(TEST_COMMITTED_BYTES_1)
+ .setTotalCapacity(TEST_TOTAL_CAPACITY)
+ .setUsedSpace(TEST_USED_SPACE_1)
+ .setEffectiveUsedSpace(TEST_EFFECTIVE_USED_SPACE_1)
+ .build(),
+ VolumeReportProto.newBuilder()
+ .setStorageId(TEST_STORAGE_ID_2)
+ .setStoragePath(TEST_STORAGE_PATH_2)
+ .setUtilization(TEST_UTILIZATION_2)
+ .setCommittedBytes(TEST_COMMITTED_BYTES_2)
+ .setTotalCapacity(TEST_TOTAL_CAPACITY)
+ .setUsedSpace(TEST_USED_SPACE_2)
+ .setEffectiveUsedSpace(TEST_EFFECTIVE_USED_SPACE_2)
+ .build()));
+
when(ozoneContainer.getDiskBalancerInfo()).thenReturn(diskBalancerInfo);
// Create datanode details
@@ -102,12 +142,29 @@ void setup() throws IOException {
@Test
void testGetDiskBalancerInfoReport() throws IOException {
- // Test REPORT type - should only return volume density
DatanodeDiskBalancerInfoProto report = server.getDiskBalancerInfo(REQUEST_WITH_OLD_CLIENT_VERSION);
-
+ VolumeReportProto volReport0 = report.getVolumeInfo(0);
+ VolumeReportProto volReport1 = report.getVolumeInfo(1);
+
assertNotNull(report);
assertNotNull(report.getNode());
assertEquals(TEST_VOLUME_DENSITY, report.getCurrentVolumeDensitySum());
+ assertEquals(TEST_IDEAL_USAGE, report.getIdealUsage());
+ assertEquals(TEST_VOLUME_INFO_COUNT, report.getVolumeInfoCount());
+ assertEquals(TEST_STORAGE_ID_1, volReport0.getStorageId());
+ assertEquals(TEST_STORAGE_PATH_1, volReport0.getStoragePath());
+ assertEquals(TEST_UTILIZATION_1, volReport0.getUtilization());
+ assertEquals(TEST_COMMITTED_BYTES_1, volReport0.getCommittedBytes());
+ assertEquals(TEST_TOTAL_CAPACITY, volReport0.getTotalCapacity());
+ assertEquals(TEST_USED_SPACE_1, volReport0.getUsedSpace());
+ assertEquals(TEST_EFFECTIVE_USED_SPACE_1, volReport0.getEffectiveUsedSpace());
+ assertEquals(TEST_STORAGE_ID_2, volReport1.getStorageId());
+ assertEquals(TEST_STORAGE_PATH_2, volReport1.getStoragePath());
+ assertEquals(TEST_UTILIZATION_2, volReport1.getUtilization());
+ assertEquals(TEST_COMMITTED_BYTES_2, volReport1.getCommittedBytes());
+ assertEquals(TEST_TOTAL_CAPACITY, volReport1.getTotalCapacity());
+ assertEquals(TEST_USED_SPACE_2, volReport1.getUsedSpace());
+ assertEquals(TEST_EFFECTIVE_USED_SPACE_2, volReport1.getEffectiveUsedSpace());
}
@Test
diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
index 2f20fde3e09..eec28fcafb2 100644
--- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto
+++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
@@ -579,6 +579,16 @@ enum DiskBalancerRunningStatus {
PAUSED = 3;
}
+message VolumeReportProto {
+ optional string storageId = 1;
+ optional string storagePath = 2;
+ optional uint64 totalCapacity = 3;
+ optional uint64 usedSpace = 4;
+ optional uint64 committedBytes = 5;
+ optional uint64 effectiveUsedSpace = 6;
+ optional double utilization = 7;
+}
+
message DatanodeDiskBalancerInfoProto {
required DatanodeDetailsProto node = 1;
required double currentVolumeDensitySum = 2;
@@ -588,4 +598,6 @@ message DatanodeDiskBalancerInfoProto {
optional uint64 failureMoveCount = 6;
optional uint64 bytesToMove = 7;
optional uint64 bytesMoved = 8;
+ optional double idealUsage = 9;
+ repeated VolumeReportProto volumeInfo = 10;
}
diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerReportSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerReportSubcommand.java
index 200c9b2cd32..62f38da74d0 100644
--- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerReportSubcommand.java
+++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerReportSubcommand.java
@@ -27,8 +27,9 @@
import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.protocol.DiskBalancerProtocol;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDiskBalancerInfoProto;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.VolumeReportProto;
+import org.apache.hadoop.util.StringUtils;
import picocli.CommandLine.Command;
/**
@@ -36,13 +37,13 @@
*/
@Command(
name = "report",
- description = "Get DiskBalancer volume density report",
+ description = "Get DiskBalancer volume density report and per volume info from datanodes",
mixinStandardHelpOptions = true,
versionProvider = HddsVersionProvider.class)
public class DiskBalancerReportSubcommand extends AbstractDiskBalancerSubCommand {
// Store reports temporarily for non-JSON mode consolidation
- private final Map<String, HddsProtos.DatanodeDiskBalancerInfoProto> reports =
+ private final Map<String, DatanodeDiskBalancerInfoProto> reports =
new ConcurrentHashMap<>();
@Override
@@ -54,7 +55,7 @@ protected Object executeCommand(String hostName) throws IOException {
// Only create JSON result object if JSON mode is enabled
if (getOptions().isJson()) {
- return createReportResult(report);
+ return toJson(report);
}
// For non-JSON mode, store the proto for later consolidation
@@ -82,36 +83,90 @@ protected void displayResults(List<String> successNodes, List<String> failedNode
// Display consolidated report for successful nodes
if (!successNodes.isEmpty() && !reports.isEmpty()) {
- List<HddsProtos.DatanodeDiskBalancerInfoProto> reportList =
- new ArrayList<>(reports.values());
+ List<DatanodeDiskBalancerInfoProto> reportList = new ArrayList<>(reports.values());
System.out.println(generateReport(reportList));
}
}
private String generateReport(List<DatanodeDiskBalancerInfoProto> protos) {
- // Sort by volume density in descending order (highest imbalance first)
- List<DatanodeDiskBalancerInfoProto> sortedProtos = new ArrayList<>(protos);
- sortedProtos.sort((a, b) ->
+ protos.sort((a, b) ->
Double.compare(b.getCurrentVolumeDensitySum(), a.getCurrentVolumeDensitySum()));
- StringBuilder formatBuilder = new StringBuilder("Report result:%n" +
- "%-60s %s%n");
-
+ StringBuilder formatBuilder = new StringBuilder("Report result:%n");
List<String> contentList = new ArrayList<>();
- contentList.add("Datanode");
- contentList.add("VolumeDensity");
-
- for (DatanodeDiskBalancerInfoProto proto : sortedProtos) {
- formatBuilder.append("%-60s %s%n");
- // Format datanode string with hostname and IP address
- String formattedDatanode = DiskBalancerSubCommandUtil.getDatanodeHostAndIp(
- proto.getNode());
- contentList.add(formattedDatanode);
- contentList.add(String.valueOf(proto.getCurrentVolumeDensitySum()));
+
+ for (int i = 0; i < protos.size(); i++) {
+ DatanodeDiskBalancerInfoProto p = protos.get(i);
+ String dn = DiskBalancerSubCommandUtil.getDatanodeHostAndIp(p.getNode());
+
+ StringBuilder header = new StringBuilder();
+ header.append("Datanode: ").append(dn).append('\n');
+ header.append("Aggregate VolumeDataDensity: ").
+ append(p.getCurrentVolumeDensitySum()).append('\n');
+
+ if (p.hasIdealUsage() && p.hasDiskBalancerConf()
+ && p.getDiskBalancerConf().hasThreshold()) {
+ double idealUsage = p.getIdealUsage();
+ double threshold = p.getDiskBalancerConf().getThreshold();
+ double lt = idealUsage - threshold / 100.0;
+ double ut = idealUsage + threshold / 100.0;
+ header.append("IdealUsage: ").append(String.format("%.8f", idealUsage));
+ header.append(" | Threshold: ").append(threshold).append('%');
+ header.append(" | ThresholdRange: (").append(String.format("%.8f", lt));
+ header.append(", ").append(String.format("%.8f", ut)).append(')').append('\n').append('\n');
+ header.append("Volume Details:").append('\n');
+ }
+ formatBuilder.append("%s%n");
+ contentList.add(header.toString());
+
+ if (p.getVolumeInfoCount() > 0 && p.hasIdealUsage()) {
+ formatBuilder.append("%-45s %-40s %15s %15s %30s %20s %15s %15s%n");
+ contentList.add("StorageID");
+ contentList.add("StoragePath");
+ contentList.add("TotalCapacity");
+ contentList.add("UsedSpace");
+ contentList.add("Container Pre-AllocatedSpace");
+ contentList.add("EffectiveUsedSpace");
+ contentList.add("Utilization");
+ contentList.add("VolumeDensity");
+
+ double ideal = p.getIdealUsage();
+ for (VolumeReportProto v : p.getVolumeInfoList()) {
+ formatBuilder.append("%-45s %-40s %15s %15s %30s %20s %15s %15s%n");
+ contentList.add(v.hasStorageId() ? v.getStorageId() : "-");
+ contentList.add(v.hasStoragePath() ? v.getStoragePath() : "-");
+ contentList.add(v.hasTotalCapacity() ? StringUtils.byteDesc(v.getTotalCapacity()) : "-");
+ contentList.add(v.hasUsedSpace() ? StringUtils.byteDesc(v.getUsedSpace()) : "-");
+ contentList.add(StringUtils.byteDesc(v.getCommittedBytes()));
+ contentList.add(v.hasEffectiveUsedSpace() ? StringUtils.byteDesc(v.getEffectiveUsedSpace()) : "-");
+ contentList.add(String.format("%.8f", v.getUtilization()));
+ contentList.add(String.format("%.8f", Math.abs(v.getUtilization() - ideal)));
+ }
+ formatBuilder.append("%n");
+ }
+
+ if (i < protos.size() - 1) {
+ formatBuilder.append("-------%n%n");
+ }
}
- return String.format(formatBuilder.toString(),
- contentList.toArray(new String[0]));
+ formatBuilder.append("%nNote:%n");
+ formatBuilder.append(" - Aggregate VolumeDataDensity: Sum of per-volume density" +
+ " (deviation from ideal); higher means more imbalance.%n");
+ formatBuilder.append(" - IdealUsage: Target utilization ratio (0-1) when volumes" +
+ " are evenly balanced.%n");
+ formatBuilder.append(" - ThresholdRange: Acceptable deviation (percent); volumes within" +
+ " IdealUsage +/- Threshold are considered balanced.%n");
+ formatBuilder.append(" - VolumeDensity: Deviation of a particular volume's utilization from IdealUsage.%n");
+ formatBuilder.append(" - Utilization: Ratio of actual used space to capacity (0-1) for a particular volume.%n");
+ formatBuilder.append(" - TotalCapacity: Total volume capacity.%n");
+ formatBuilder.append(" - UsedSpace: Ozone used space.%n");
+ formatBuilder.append(" - Container Pre-AllocatedSpace: Space reserved for containers not yet written to disk.%n");
+ formatBuilder.append(" - EffectiveUsedSpace: This is the actual used space of volume which is visible" +
+ " to the diskBalancer : (ozoneCapacity minus ozoneAvailable) + containerPreAllocatedSpace + " +
+ "move delta for source volume.%n");
+
+ return String.format(formatBuilder.toString(), contentList.toArray(new String[0]));
}
@Override
@@ -121,20 +176,47 @@ protected String getActionName() {
/**
* Create a JSON result map for a report.
- *
+ *
* @param report the DiskBalancer report proto
* @return JSON result map
*/
- private Map<String, Object> createReportResult(
- HddsProtos.DatanodeDiskBalancerInfoProto report) {
+ private Map<String, Object> toJson(DatanodeDiskBalancerInfoProto report) {
Map<String, Object> result = new LinkedHashMap<>();
- // Format datanode string with hostname and IP address
- String formattedDatanode = DiskBalancerSubCommandUtil.getDatanodeHostAndIp(
- report.getNode());
- result.put("datanode", formattedDatanode);
+ result.put("datanode", DiskBalancerSubCommandUtil.getDatanodeHostAndIp(report.getNode()));
result.put("action", "report");
result.put("status", "success");
result.put("volumeDensity", report.getCurrentVolumeDensitySum());
+
+ if (report.hasIdealUsage() && report.hasDiskBalancerConf()
+ && report.getDiskBalancerConf().hasThreshold()) {
+ double idealUsage = report.getIdealUsage();
+ double threshold = report.getDiskBalancerConf().getThreshold();
+ double lt = idealUsage - threshold / 100.0;
+ double ut = idealUsage + threshold / 100.0;
+ result.put("idealUsage", String.format("%.8f", idealUsage));
+ result.put("threshold %", report.getDiskBalancerConf().getThreshold());
+ result.put("thresholdRange", String.format("(%.08f, %.08f)", lt, ut));
+ }
+
+ if (report.getVolumeInfoCount() > 0) {
+ double ideal = report.hasIdealUsage() ? report.getIdealUsage() : 0.0;
+ List<Map<String, Object>> vols = new ArrayList<>();
+ for (VolumeReportProto v : report.getVolumeInfoList()) {
+ Map<String, Object> vm = new LinkedHashMap<>();
+ vm.put("storageId", v.getStorageId());
+ vm.put("storagePath", v.hasStoragePath() ? v.getStoragePath() : "-");
+ vm.put("totalCapacity", v.hasTotalCapacity() ? StringUtils.byteDesc(v.getTotalCapacity()) : "-");
+ vm.put("usedSpace", v.hasUsedSpace() ? StringUtils.byteDesc(v.getUsedSpace()) : "-");
+ vm.put("containerPreAllocatedSpace", StringUtils.byteDesc(v.getCommittedBytes()));
+ vm.put("effectiveUsedSpace", v.hasEffectiveUsedSpace() ?
+ StringUtils.byteDesc(v.getEffectiveUsedSpace()) : "-");
+ vm.put("utilization", v.getUtilization());
+ vm.put("volumeDensity", Math.abs(v.getUtilization() - ideal));
+ vols.add(vm);
+ }
+
+ result.put("volumes", vols);
+ }
return result;
}
}
diff --git a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestDiskBalancerSubCommands.java b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestDiskBalancerSubCommands.java
index 76040867920..893fd6ef013 100644
--- a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestDiskBalancerSubCommands.java
+++ b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestDiskBalancerSubCommands.java
@@ -51,6 +51,7 @@
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDiskBalancerInfoProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DiskBalancerConfigurationProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DiskBalancerRunningStatus;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.VolumeReportProto;
import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@@ -615,6 +616,16 @@ public void testReportDiskBalancerWithJson() throws Exception {
String output = outContent.toString(DEFAULT_ENCODING);
assertTrue(output.contains("\"datanode\""));
assertTrue(output.contains("\"volumeDensity\""));
+ assertTrue(output.contains("\"idealUsage\""));
+ assertTrue(output.contains("\"volumes\""));
+ assertTrue(output.contains("\"storageId\""));
+ assertTrue(output.contains("\"storagePath\""));
+ assertTrue(output.contains("\"totalCapacity\""));
+ assertTrue(output.contains("\"usedSpace\""));
+ assertTrue(output.contains("\"effectiveUsedSpace\""));
+ assertTrue(output.contains("\"utilization\""));
+ assertTrue(output.contains("\"volumeDensity\""));
+ assertTrue(output.contains("\"containerPreAllocatedSpace\""));
}
}
@@ -699,12 +710,8 @@ private DatanodeDiskBalancerInfoProto createStatusProto(String hostname,
.build())
.build();
- DiskBalancerConfigurationProto configProto = DiskBalancerConfigurationProto.newBuilder()
- .setThreshold(threshold)
- .setDiskBandwidthInMB(bandwidthInMB)
- .setParallelThread(parallelThread)
- .setStopAfterDiskEven(true)
- .build();
+ DiskBalancerConfigurationProto configProto = createConfigProto(threshold, bandwidthInMB, parallelThread, true);
+
return DatanodeDiskBalancerInfoProto.newBuilder()
.setNode(nodeProto)
@@ -741,10 +748,12 @@ private DatanodeDiskBalancerInfoProto generateRandomStatusProto(String hostname)
/**
* Generates a random report proto for a given hostname.
* @param hostname the hostname
- * @return DatanodeDiskBalancerInfoProto with random volume density
+ * @return DatanodeDiskBalancerInfoProto with random volume density and volume info
*/
private DatanodeDiskBalancerInfoProto generateRandomReportProto(String hostname) {
double volumeDensity = random.nextDouble() * 0.1;
+ double idealUsage = 0.1 + random.nextDouble() * 0.3;
+ double threshold = 5.0 + random.nextDouble() * 10.0;
DatanodeDetailsProto nodeProto = DatanodeDetailsProto.newBuilder()
.setHostName(hostname)
.setIpAddress("127.0.0.1")
@@ -754,9 +763,56 @@ private DatanodeDiskBalancerInfoProto generateRandomReportProto(String hostname)
.build())
.build();
+ DiskBalancerConfigurationProto configProto = createConfigProto(threshold, 100L, 5, true);
+
+ long committed1 = (random.nextLong() & Long.MAX_VALUE) % (1024L * 1024 * 1024);
+ long committed2 = (random.nextLong() & Long.MAX_VALUE) % (1024L * 1024 * 1024);
+ long capacity1 = 500L * 1024 * 1024 * 1024 + random.nextInt(1024 * 1024);
+ long capacity2 = 600L * 1024 * 1024 * 1024 + random.nextInt(1024 * 1024);
+ double util1 = idealUsage + random.nextDouble() * 0.1;
+ double util2 = idealUsage - random.nextDouble() * 0.1;
+ long used1 = (long) (capacity1 * util1);
+ long used2 = (long) (capacity2 * util2);
+ long effective1 = used1 + committed1;
+ long effective2 = used2 + committed2;
+ String path1 = "/data/hdds-" + hostname + "-1";
+ String path2 = "/data/hdds-" + hostname + "-2";
+ VolumeReportProto vol1 = VolumeReportProto.newBuilder()
+ .setStorageId("DISK-" + hostname + "-vol1")
+ .setStoragePath(path1)
+ .setUtilization(util1)
+ .setCommittedBytes(committed1)
+ .setTotalCapacity(capacity1)
+ .setUsedSpace(used1)
+ .setEffectiveUsedSpace(effective1)
+ .build();
+ VolumeReportProto vol2 = VolumeReportProto.newBuilder()
+ .setStorageId("DISK-" + hostname + "-vol2")
+ .setStoragePath(path2)
+ .setUtilization(util2)
+ .setCommittedBytes(committed2)
+ .setTotalCapacity(capacity2)
+ .setUsedSpace(used2)
+ .setEffectiveUsedSpace(effective2)
+ .build();
+
return DatanodeDiskBalancerInfoProto.newBuilder()
.setNode(nodeProto)
.setCurrentVolumeDensitySum(volumeDensity)
+ .setIdealUsage(idealUsage)
+ .setDiskBalancerConf(configProto)
+ .addVolumeInfo(vol1)
+ .addVolumeInfo(vol2)
+ .build();
+ }
+
+ private DiskBalancerConfigurationProto createConfigProto(double threshold, long bandwidthInMB, int parallelThread,
+ boolean stopAfterDiskEven) {
+ return DiskBalancerConfigurationProto.newBuilder()
+ .setThreshold(threshold)
+ .setDiskBandwidthInMB(bandwidthInMB)
+ .setParallelThread(parallelThread)
+ .setStopAfterDiskEven(stopAfterDiskEven)
.build();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org