You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by sa...@apache.org on 2026/03/17 12:10:03 UTC
(ozone) branch master updated: HDDS-14701. Consolidate DiskBalancerVolumeChoosingPolicy and ContainerChoosingPolicy (#9858)
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 020452e196b HDDS-14701. Consolidate DiskBalancerVolumeChoosingPolicy and ContainerChoosingPolicy (#9858)
020452e196b is described below
commit 020452e196bbd0de0d530f6b2291339bacc66213
Author: Gargi Jaiswal <13...@users.noreply.github.com>
AuthorDate: Tue Mar 17 17:39:56 2026 +0530
HDDS-14701. Consolidate DiskBalancerVolumeChoosingPolicy and ContainerChoosingPolicy (#9858)
---
.../common/volume/VolumeChoosingPolicyFactory.java | 11 +-
.../ContainerChoosingPolicyFactory.java | 55 ++
.../diskbalancer/DiskBalancerConfiguration.java | 18 +-
.../diskbalancer/DiskBalancerService.java | 70 +--
...ChoosingPolicy.java => ContainerCandidate.java} | 42 +-
.../policy/ContainerChoosingPolicy.java | 33 +-
.../policy/DefaultContainerChoosingPolicy.java | 149 ++++-
.../policy/DefaultVolumeChoosingPolicy.java | 156 -----
.../TestDefaultContainerChoosingPolicy.java | 700 +++++++++++++++++----
.../TestDefaultVolumeChoosingPolicy.java | 651 -------------------
.../diskbalancer/TestDiskBalancerService.java | 18 +-
.../diskbalancer/TestDiskBalancerTask.java | 10 +-
hadoop-hdds/docs/content/design/diskbalancer.md | 14 +-
hadoop-hdds/docs/content/feature/DiskBalancer.md | 23 +-
.../docs/content/feature/DiskBalancer.zh.md | 23 +-
.../scm/node/TestContainerChoosingPolicy.java | 300 ---------
.../node/TestDiskBalancerPolicyPerformance.java | 376 +++++++++++
.../ozone/scm/node/TestVolumeChoosingPolicy.java | 285 ---------
18 files changed, 1242 insertions(+), 1692 deletions(-)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeChoosingPolicyFactory.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeChoosingPolicyFactory.java
index e4c8aa3b514..90f7f7753cf 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeChoosingPolicyFactory.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeChoosingPolicyFactory.java
@@ -23,8 +23,6 @@
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy;
-import org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerConfiguration;
-import org.apache.hadoop.ozone.container.diskbalancer.policy.DiskBalancerVolumeChoosingPolicy;
import org.apache.ratis.util.ReflectionUtils;
/**
@@ -48,9 +46,10 @@ public static VolumeChoosingPolicy getPolicy(ConfigurationSource conf) {
return ReflectionUtils.newInstance(policyClass, new Class<?>[] {ReentrantLock.class}, LOCK);
}
- public static DiskBalancerVolumeChoosingPolicy getDiskBalancerPolicy(ConfigurationSource conf) {
- Class<?> policyClass = conf.getObject(DiskBalancerConfiguration.class).getVolumeChoosingPolicyClass();
- return (DiskBalancerVolumeChoosingPolicy) ReflectionUtils.newInstance(
- policyClass, new Class<?>[]{ReentrantLock.class}, LOCK);
+ /**
+ * Returns the shared lock used for volume space reservation.
+ */
+ public static ReentrantLock getVolumeSpaceReservationLock() {
+ return LOCK;
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/ContainerChoosingPolicyFactory.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/ContainerChoosingPolicyFactory.java
new file mode 100644
index 00000000000..652e0c29781
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/ContainerChoosingPolicyFactory.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.diskbalancer;
+
+import static org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerConfiguration.HDDS_DATANODE_DISKBALANCER_CONTAINER_CHOOSING_POLICY;
+
+import java.util.concurrent.locks.ReentrantLock;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.ozone.container.common.volume.VolumeChoosingPolicyFactory;
+import org.apache.hadoop.ozone.container.diskbalancer.policy.ContainerChoosingPolicy;
+import org.apache.hadoop.ozone.container.diskbalancer.policy.DefaultContainerChoosingPolicy;
+import org.apache.ratis.util.ReflectionUtils;
+
+/**
+ * A factory to create {@link ContainerChoosingPolicy} instances for the DiskBalancer.
+ * The policy class is configured via
+ * {@link DiskBalancerConfiguration#getContainerChoosingPolicyClass()}.
+ */
+public final class ContainerChoosingPolicyFactory {
+
+ private static final Class<? extends ContainerChoosingPolicy>
+ DEFAULT_CONTAINER_CHOOSING_POLICY = DefaultContainerChoosingPolicy.class;
+
+ private ContainerChoosingPolicyFactory() {
+ }
+
+ /**
+ * Creates a ContainerChoosingPolicy instance from configuration.
+ *
+ * @param conf the configuration source
+ * @return a configured ContainerChoosingPolicy instance
+ */
+ public static ContainerChoosingPolicy getDiskBalancerPolicy(ConfigurationSource conf) {
+ Class<? extends ContainerChoosingPolicy> policyClass = conf.getClass(
+ HDDS_DATANODE_DISKBALANCER_CONTAINER_CHOOSING_POLICY,
+ DEFAULT_CONTAINER_CHOOSING_POLICY, ContainerChoosingPolicy.class);
+ return ReflectionUtils.newInstance(policyClass, new Class<?>[]{ReentrantLock.class},
+ VolumeChoosingPolicyFactory.getVolumeSpaceReservationLock());
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerConfiguration.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerConfiguration.java
index 447691c0a5b..9f35a1f3edf 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerConfiguration.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerConfiguration.java
@@ -94,19 +94,15 @@ public final class DiskBalancerConfiguration {
)
private long diskBalancerTimeout = Duration.ofSeconds(300).toMillis();
- @Config(key = "hdds.datanode.disk.balancer.volume.choosing.policy", type = ConfigType.CLASS,
- defaultValue = "org.apache.hadoop.ozone.container.diskbalancer.policy" +
- ".DefaultVolumeChoosingPolicy",
- tags = {ConfigTag.DISKBALANCER},
- description = "The volume choosing policy of the disk balancer service.")
- private Class<?> volumeChoosingPolicyClass;
+ public static final String HDDS_DATANODE_DISKBALANCER_CONTAINER_CHOOSING_POLICY =
+ "hdds.datanode.disk.balancer.container.choosing.policy";
- @Config(key = "hdds.datanode.disk.balancer.container.choosing.policy", type = ConfigType.CLASS,
+ @Config(key = HDDS_DATANODE_DISKBALANCER_CONTAINER_CHOOSING_POLICY, type = ConfigType.CLASS,
defaultValue = "org.apache.hadoop.ozone.container.diskbalancer.policy" +
".DefaultContainerChoosingPolicy",
tags = {ConfigTag.DISKBALANCER},
- description = "The container choosing policy of the disk balancer " +
- "service.")
+ description = "The policy for selecting source/destination volumes and " +
+ "containers to move for disk balancing.")
private Class<?> containerChoosingPolicyClass;
@Config(key = "hdds.datanode.disk.balancer.stop.after.disk.even",
@@ -174,10 +170,6 @@ public void setDiskBalancerTimeout(Duration duration) {
this.diskBalancerTimeout = duration.toMillis();
}
- public Class<?> getVolumeChoosingPolicyClass() {
- return volumeChoosingPolicyClass;
- }
-
public Class<?> getContainerChoosingPolicyClass() {
return containerChoosingPolicyClass;
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
index 9503c2e3c1f..596cf87fde3 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
@@ -41,15 +41,12 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
-import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.fs.SpaceUsageSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DiskBalancerRunningStatus;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.hdds.server.ServerUtils;
@@ -67,11 +64,10 @@
import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
-import org.apache.hadoop.ozone.container.common.volume.VolumeChoosingPolicyFactory;
import org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerVolumeCalculation.VolumeFixedUsage;
+import org.apache.hadoop.ozone.container.diskbalancer.policy.ContainerCandidate;
import org.apache.hadoop.ozone.container.diskbalancer.policy.ContainerChoosingPolicy;
import org.apache.hadoop.ozone.container.diskbalancer.policy.DefaultContainerChoosingPolicy;
-import org.apache.hadoop.ozone.container.diskbalancer.policy.DiskBalancerVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerLocationUtil;
import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil;
@@ -126,12 +122,10 @@ public class DiskBalancerService extends BackgroundService {
private Map<HddsVolume, Long> deltaSizes;
private MutableVolumeSet volumeSet;
- private DiskBalancerVolumeChoosingPolicy volumeChoosingPolicy;
- private ContainerChoosingPolicy containerChoosingPolicy;
+ private ContainerChoosingPolicy volumeContainerChoosingPolicy;
private final File diskBalancerInfoFile;
private DiskBalancerServiceMetrics metrics;
- private long containerDefaultSize;
public DiskBalancerService(OzoneContainer ozoneContainer,
long serviceCheckInterval, long serviceCheckTimeout, TimeUnit timeUnit,
@@ -148,15 +142,9 @@ public DiskBalancerService(OzoneContainer ozoneContainer,
inProgressContainers = ConcurrentHashMap.newKeySet();
deltaSizes = new ConcurrentHashMap<>();
volumeSet = ozoneContainer.getVolumeSet();
- containerDefaultSize = (long) conf.getStorageSize(
- ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
- ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
try {
- volumeChoosingPolicy = VolumeChoosingPolicyFactory.getDiskBalancerPolicy(conf);
- containerChoosingPolicy = (ContainerChoosingPolicy)
- conf.getObject(DiskBalancerConfiguration.class)
- .getContainerChoosingPolicyClass().newInstance();
+ volumeContainerChoosingPolicy = ContainerChoosingPolicyFactory.getDiskBalancerPolicy(conf);
} catch (Exception e) {
LOG.error("Got exception when initializing DiskBalancerService", e);
throw new IOException(e);
@@ -405,25 +393,20 @@ public BackgroundTaskQueue getTasks() {
}
for (int i = 0; i < availableTaskCount; i++) {
- Pair<HddsVolume, HddsVolume> pair = volumeChoosingPolicy
- .chooseVolume(volumeSet, threshold, deltaSizes, containerDefaultSize);
- if (pair == null) {
- continue;
- }
- HddsVolume sourceVolume = pair.getLeft(), destVolume = pair.getRight();
- ContainerData toBalanceContainer = containerChoosingPolicy
- .chooseContainer(ozoneContainer, sourceVolume, destVolume,
- inProgressContainers, threshold, volumeSet, deltaSizes);
- if (toBalanceContainer != null) {
- DiskBalancerTask task = new DiskBalancerTask(toBalanceContainer, sourceVolume,
- destVolume);
- queue.add(task);
- inProgressContainers.add(ContainerID.valueOf(toBalanceContainer.getContainerID()));
- deltaSizes.put(sourceVolume, deltaSizes.getOrDefault(sourceVolume, 0L)
- - toBalanceContainer.getBytesUsed());
- } else {
- // release destVolume committed bytes
- destVolume.incCommittedBytes(0 - containerDefaultSize);
+ ContainerCandidate candidate = volumeContainerChoosingPolicy.chooseVolumesAndContainer(
+ ozoneContainer, volumeSet, deltaSizes, inProgressContainers, threshold);
+ if (candidate != null) {
+ HddsVolume sourceVolume = candidate.getSourceVolume();
+ HddsVolume destVolume = candidate.getDestVolume();
+ ContainerData toBalanceContainer = candidate.getContainerData();
+ if (toBalanceContainer != null) {
+ DiskBalancerTask task = new DiskBalancerTask(toBalanceContainer, sourceVolume,
+ destVolume);
+ queue.add(task);
+ inProgressContainers.add(ContainerID.valueOf(toBalanceContainer.getContainerID()));
+ deltaSizes.put(sourceVolume, deltaSizes.getOrDefault(sourceVolume, 0L)
+ - toBalanceContainer.getBytesUsed());
+ }
}
}
@@ -640,7 +623,7 @@ private void postCall(boolean success, long startTime) {
inProgressContainers.remove(ContainerID.valueOf(containerData.getContainerID()));
deltaSizes.put(sourceVolume, deltaSizes.get(sourceVolume) +
containerData.getBytesUsed());
- destVolume.incCommittedBytes(0 - containerDefaultSize);
+ destVolume.incCommittedBytes(0 - containerData.getBytesUsed());
long endTime = Time.monotonicNow();
if (success) {
metrics.incrSuccessCount(1);
@@ -749,22 +732,13 @@ public void setBalancedBytesInLastWindow(long bytes) {
this.balancedBytesInLastWindow.set(bytes);
}
- public ContainerChoosingPolicy getContainerChoosingPolicy() {
- return containerChoosingPolicy;
- }
-
- public DiskBalancerVolumeChoosingPolicy getVolumeChoosingPolicy() {
- return volumeChoosingPolicy;
- }
-
- @VisibleForTesting
- public void setVolumeChoosingPolicy(DiskBalancerVolumeChoosingPolicy volumeChoosingPolicy) {
- this.volumeChoosingPolicy = volumeChoosingPolicy;
+ public ContainerChoosingPolicy getVolumeContainerChoosingPolicy() {
+ return volumeContainerChoosingPolicy;
}
@VisibleForTesting
- public void setContainerChoosingPolicy(ContainerChoosingPolicy containerChoosingPolicy) {
- this.containerChoosingPolicy = containerChoosingPolicy;
+ public void setVolumeContainerChoosingPolicy(ContainerChoosingPolicy volumeContainerChoosingPolicy) {
+ this.volumeContainerChoosingPolicy = volumeContainerChoosingPolicy;
}
@VisibleForTesting
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DiskBalancerVolumeChoosingPolicy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/ContainerCandidate.java
similarity index 53%
rename from hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DiskBalancerVolumeChoosingPolicy.java
rename to hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/ContainerCandidate.java
index c64733fc37e..615f9c3bbc4 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DiskBalancerVolumeChoosingPolicy.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/ContainerCandidate.java
@@ -17,24 +17,34 @@
package org.apache.hadoop.ozone.container.diskbalancer.policy;
-import java.util.Map;
-import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
-import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
/**
- * This interface specifies the policy for choosing volumes to balance.
+ * Result of consolidated volume and container selection for disk balancing.
+ * Contains the container to move and its source and destination volumes.
*/
-public interface DiskBalancerVolumeChoosingPolicy {
- /**
- * Choose a pair of volumes for balancing.
- *
- * @param volumeSet - volumes to choose from.
- * @param thresholdPercentage the threshold percentage in range (0, 100) to choose the source volume.
- * @param deltaSizes - the sizes changes of inProgress balancing jobs.
- * @param containerSize - the estimated size of container to be moved.
- * @return Source volume and Dest volume.
- */
- Pair<HddsVolume, HddsVolume> chooseVolume(MutableVolumeSet volumeSet,
- double thresholdPercentage, Map<HddsVolume, Long> deltaSizes, long containerSize);
+public final class ContainerCandidate {
+ private final ContainerData containerData;
+ private final HddsVolume sourceVolume;
+ private final HddsVolume destVolume;
+
+ public ContainerCandidate(ContainerData containerData,
+ HddsVolume sourceVolume, HddsVolume destVolume) {
+ this.containerData = containerData;
+ this.sourceVolume = sourceVolume;
+ this.destVolume = destVolume;
+ }
+
+ public ContainerData getContainerData() {
+ return containerData;
+ }
+
+ public HddsVolume getSourceVolume() {
+ return sourceVolume;
+ }
+
+ public HddsVolume getDestVolume() {
+ return destVolume;
+ }
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/ContainerChoosingPolicy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/ContainerChoosingPolicy.java
index d512de84bdf..c00588d55b7 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/ContainerChoosingPolicy.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/ContainerChoosingPolicy.java
@@ -20,30 +20,33 @@
import java.util.Map;
import java.util.Set;
import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
/**
- * This interface specifies the policy for choosing containers to balance.
+ * This interface specifies the policy for choosing volumes and containers to balance.
+ * It provides consolidated volume selection (source/destination pair) and container selection
+ * into a single operation to avoid recalculating ideal utilization and disk usage.
*/
public interface ContainerChoosingPolicy {
/**
- * Choose a container for balancing.
- * @param ozoneContainer the OzoneContainer instance to get all containers of a particular volume.
- * @param srcVolume the HddsVolume instance to choose containers from.
- * @param destVolume the destination volume to which container is being moved.
- * @param inProgressContainerIDs containerIDs present in this set should be
- - avoided as these containers are already under move by diskBalancer.
- * @param thresholdPercentage the threshold percentage in range (0, 100)
+ * Choose a container and its source/destination volumes for balancing.
+ * Performs both volume pair selection and container selection in one call,
+ * computing ideal usage and volume utilizations only once.
+ * Space is reserved on the destination only when a container is chosen,
+ * using the actual container size.
+ *
+ * @param ozoneContainer the OzoneContainer instance to get all containers
* @param volumeSet the volumeSet instance
- * @param deltaMap the deltaMap instance of source volume
- * @return a Container
+ * @param deltaMap the deltaMap for in-progress balancing jobs (negative = space to be freed)
+ * @param inProgressContainerIDs containerIDs to avoid (already under move)
+ * @param thresholdPercentage the threshold percentage in range (0, 100)
+ * @return a DiskBalancerVolumeContainerCandidate with container and volumes, or null if none found
*/
- ContainerData chooseContainer(OzoneContainer ozoneContainer,
- HddsVolume srcVolume, HddsVolume destVolume,
+ ContainerCandidate chooseVolumesAndContainer(OzoneContainer ozoneContainer,
+ MutableVolumeSet volumeSet,
+ Map<HddsVolume, Long> deltaMap,
Set<ContainerID> inProgressContainerIDs,
- double thresholdPercentage, MutableVolumeSet volumeSet,
- Map<HddsVolume, Long> deltaMap);
+ double thresholdPercentage);
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultContainerChoosingPolicy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultContainerChoosingPolicy.java
index f964f6f519e..5ead73db968 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultContainerChoosingPolicy.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultContainerChoosingPolicy.java
@@ -20,29 +20,36 @@
import static java.util.concurrent.TimeUnit.HOURS;
import static org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerVolumeCalculation.computeUtilization;
import static org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerVolumeCalculation.getIdealUsage;
-import static org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerVolumeCalculation.getVolumeUsages;
+import static org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerVolumeCalculation.newVolumeFixedUsage;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
+import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
import org.apache.hadoop.hdds.fs.SpaceUsageSource;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
+import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
import org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerVolumeCalculation.VolumeFixedUsage;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Choose a container from specified volume, make sure it's not being balancing.
+ * First chooses a source volume and destination volume pair based on ideal utilization and threshold,
+ * then chooses a container from the source volume that can be moved to the destination without
+ * exceeding the upper threshold. Space is reserved on the destination only when a container is
+ * chosen, using the actual container size.
*/
public class DefaultContainerChoosingPolicy implements ContainerChoosingPolicy {
public static final Logger LOG = LoggerFactory.getLogger(
@@ -55,12 +62,96 @@ public class DefaultContainerChoosingPolicy implements ContainerChoosingPolicy {
// for test
private static boolean test = false;
+ private final ReentrantLock lock;
+
+ public DefaultContainerChoosingPolicy(ReentrantLock globalLock) {
+ this.lock = globalLock;
+ }
+
@Override
- public ContainerData chooseContainer(OzoneContainer ozoneContainer,
- HddsVolume src, HddsVolume dst,
- Set<ContainerID> inProgressContainerIDs,
- double thresholdPercentage, MutableVolumeSet volumeSet,
- Map<HddsVolume, Long> deltaMap) {
+ public ContainerCandidate chooseVolumesAndContainer(OzoneContainer ozoneContainer,
+ MutableVolumeSet volumeSet, Map<HddsVolume, Long> deltaMap, Set<ContainerID> inProgressContainerIDs,
+ double thresholdPercentage) {
+ lock.lock();
+ try {
+ // Create truly immutable snapshot of volumes to ensure consistency
+ final List<StorageVolume> allVolumes = volumeSet.getVolumesList();
+ if (allVolumes.size() < 2) {
+ return null; // Can't balance with less than 2 volumes.
+ }
+
+ // Calculate usages and sort in ascending order of utilization (once)
+ // Use storage ID as secondary sort for deterministic ordering when utilizations are equal
+ final List<VolumeFixedUsage> volumeUsages = allVolumes.stream()
+ .map(v -> newVolumeFixedUsage(v, deltaMap))
+ .sorted(Comparator.comparingDouble(VolumeFixedUsage::getUtilization)
+ .thenComparing(v -> v.getVolume().getStorageID()))
+ .collect(Collectors.toList());
+
+ // Calculate ideal usage and threshold range (once)
+ final double idealUsage = getIdealUsage(volumeUsages);
+ final double actualThreshold = thresholdPercentage / 100.0;
+ final double lowerThreshold = idealUsage - actualThreshold;
+ final double upperThreshold = idealUsage + actualThreshold;
+
+ if (LOG.isDebugEnabled()) {
+ logVolumeBalancingState(volumeUsages, idealUsage, thresholdPercentage,
+ lowerThreshold, upperThreshold, deltaMap);
+ }
+
+ // Get highest and lowest utilization volumes
+ final VolumeFixedUsage highestUsage = volumeUsages.get(volumeUsages.size() - 1);
+ final VolumeFixedUsage lowestUsage = volumeUsages.get(0);
+
+ // Only return null if highest is below upper threshold AND lowest is above lower threshold
+ if (highestUsage.getUtilization() < upperThreshold &&
+ lowestUsage.getUtilization() > lowerThreshold) {
+ return null;
+ }
+
+ // Determine source volume: highest utilization volume
+ final VolumeFixedUsage srcUsage = highestUsage;
+ final HddsVolume src = srcUsage.getVolume();
+
+ // Find destination volume and container: try each dest with lower utilization than source
+ for (int i = 0; i < volumeUsages.size() - 1; i++) {
+ final VolumeFixedUsage dstUsage = volumeUsages.get(i);
+ final HddsVolume dst = dstUsage.getVolume();
+
+ // Check if destination has lower utilization than source and some usable space
+ if (dstUsage.getUtilization() < srcUsage.getUtilization() &&
+ dstUsage.computeUsableSpace() > 0) {
+ ContainerData containerData = chooseContainer(ozoneContainer,
+ src, dst, dstUsage, inProgressContainerIDs, upperThreshold);
+ if (containerData != null) {
+ long containerSize = containerData.getBytesUsed();
+ dst.incCommittedBytes(containerSize);
+ LOG.debug("Chosen volume pair for disk balancing: source={} (utilization={}), "
+ + "destination={} (utilization={})",
+ src.getStorageID(), srcUsage.getUtilization(),
+ dst.getStorageID(), dstUsage.getUtilization());
+ return new ContainerCandidate(containerData, src, dst);
+ }
+ LOG.debug("No suitable container found for destination {}, trying next volume.",
+ dst.getStorageID());
+ } else {
+ LOG.debug("Destination volume {} does not have enough space, trying next volume.",
+ dst.getStorageID());
+ }
+ }
+ LOG.debug("Failed to find appropriate destination volume and container.");
+ return null;
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ /**
+ * Choose a container from source volume that can be moved to destination volume.
+ */
+ private ContainerData chooseContainer(OzoneContainer ozoneContainer,
+ HddsVolume src, HddsVolume dst, VolumeFixedUsage dstUsage,
+ Set<ContainerID> inProgressContainerIDs, double upperThreshold) {
final Iterator<Container<?>> itr;
try {
itr = CACHE.get().get(src, () -> ozoneContainer.getController().getContainers(src));
@@ -69,21 +160,26 @@ public ContainerData chooseContainer(OzoneContainer ozoneContainer,
return null;
}
- // Calculate the actual threshold
- final List<VolumeFixedUsage> volumeUsages = getVolumeUsages(volumeSet, deltaMap);
- final double actualThreshold = getIdealUsage(volumeUsages) + thresholdPercentage / 100.0;
-
- // Find container
- final SpaceUsageSource.Fixed dstUsage = dst.getCurrentUsage();
+ final SpaceUsageSource.Fixed dstSpaceUsage = dstUsage.getUsage();
final long dstCommittedBytes = dst.getCommittedBytes();
+ final long usableSpace = dstUsage.computeUsableSpace();
+
while (itr.hasNext()) {
ContainerData containerData = itr.next().getContainerData();
+
+ // Skip containers removed from containerSet after iterator was cached
+ if (ozoneContainer.getContainerSet().getContainer(containerData.getContainerID()) == null) {
+ continue;
+ }
+
if (containerData.getBytesUsed() > 0 &&
!inProgressContainerIDs.contains(ContainerID.valueOf(containerData.getContainerID())) &&
(containerData.isClosed() || (test && containerData.isQuasiClosed()))) {
- // Check if dst can accept the candidate container.
- if (computeUtilization(dstUsage, dstCommittedBytes, containerData.getBytesUsed()) < actualThreshold) {
+ long containerSize = containerData.getBytesUsed();
+ // Check if dst has enough space and can accept the container without exceeding threshold
+ if (containerSize < usableSpace &&
+ computeUtilization(dstSpaceUsage, dstCommittedBytes, containerSize) < upperThreshold) {
return containerData;
}
}
@@ -93,6 +189,29 @@ public ContainerData chooseContainer(OzoneContainer ozoneContainer,
return null;
}
+ /**
+ * Logs all volume information for disk balancing investigation.
+ */
+ private void logVolumeBalancingState(List<VolumeFixedUsage> volumeUsages,
+ double idealUsage, double thresholdPercentage, double lowerThreshold,
+ double upperThreshold, Map<HddsVolume, Long> deltaMap) {
+ LOG.debug("Disk balancing state - idealUsage={}, thresholdPercentage={}%, "
+ + "thresholdRange=({}, {})",
+ String.format("%.10f", idealUsage), thresholdPercentage,
+ String.format("%.10f", lowerThreshold), String.format("%.10f", upperThreshold));
+ for (int i = 0; i < volumeUsages.size(); i++) {
+ VolumeFixedUsage vfu = volumeUsages.get(i);
+ HddsVolume vol = vfu.getVolume();
+ SpaceUsageSource.Fixed usage = vfu.getUsage();
+ long usableSpace = vfu.computeUsableSpace();
+ LOG.debug("Volume[{}] - disk={}, utilization={}, capacity={}, "
+ + "effectiveUsed={}, available={}, usableSpace={}, committedBytes={}, delta={}",
+ i, vol.getStorageID(), String.format("%.10f", vfu.getUtilization()),
+ usage.getCapacity(), vfu.getEffectiveUsed(), usage.getAvailable(),
+ usableSpace, vol.getCommittedBytes(), deltaMap.getOrDefault(vol, 0L));
+ }
+ }
+
@VisibleForTesting
public static void setTest(boolean isTest) {
test = isTest;
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultVolumeChoosingPolicy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultVolumeChoosingPolicy.java
deleted file mode 100644
index 240c5a9f9f0..00000000000
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultVolumeChoosingPolicy.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.diskbalancer.policy;
-
-import static org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerVolumeCalculation.getIdealUsage;
-import static org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerVolumeCalculation.newVolumeFixedUsage;
-
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.stream.Collectors;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.hadoop.hdds.fs.SpaceUsageSource;
-import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
-import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
-import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
-import org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerVolumeCalculation.VolumeFixedUsage;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Choose a random volume for disk balancing.
- *
- * Source volumes use deltaMap to simulate space that will be freed (pre-deleted).
- * Destination volumes use committedBytes to account for space already reserved.
- * Both deltaMap and committedBytes are considered to calculate usage.
- */
-public class DefaultVolumeChoosingPolicy implements DiskBalancerVolumeChoosingPolicy {
-
- public static final Logger LOG = LoggerFactory.getLogger(
- DefaultVolumeChoosingPolicy.class);
- private final ReentrantLock lock;
-
- public DefaultVolumeChoosingPolicy(ReentrantLock globalLock) {
- lock = globalLock;
- }
-
- @Override
- public Pair<HddsVolume, HddsVolume> chooseVolume(MutableVolumeSet volumeSet,
- double thresholdPercentage, Map<HddsVolume, Long> deltaMap, long containerSize) {
- lock.lock();
- try {
- // Create truly immutable snapshot of volumes to ensure consistency
- final List<StorageVolume> allVolumes = volumeSet.getVolumesList();
- if (allVolumes.size() < 2) {
- return null; // Can't balance with less than 2 volumes.
- }
-
- // Calculate usages and sort in ascending order of utilization
- final List<VolumeFixedUsage> volumeUsages = allVolumes.stream()
- .map(v -> newVolumeFixedUsage(v, deltaMap))
- .sorted(Comparator.comparingDouble(VolumeFixedUsage::getUtilization))
- .collect(Collectors.toList());
-
- // Calculate ideal usage and threshold range
- final double idealUsage = getIdealUsage(volumeUsages);
- final double actualThreshold = thresholdPercentage / 100.0;
- final double lowerThreshold = idealUsage - actualThreshold;
- final double upperThreshold = idealUsage + actualThreshold;
-
- // Log all volume information for investigation
- if (LOG.isDebugEnabled()) {
- logVolumeBalancingState(volumeUsages, idealUsage, thresholdPercentage,
- lowerThreshold, upperThreshold, containerSize, deltaMap);
- }
-
- // Get highest and lowest utilization volumes
- final VolumeFixedUsage highestUsage = volumeUsages.get(volumeUsages.size() - 1);
- final VolumeFixedUsage lowestUsage = volumeUsages.get(0);
-
- // Only return null if highest is below upper threshold AND lowest is above lower threshold
- // This means all volumes are strictly within the range (not at boundaries)
- if (highestUsage.getUtilization() < upperThreshold &&
- lowestUsage.getUtilization() > lowerThreshold) {
- // All volumes are strictly within threshold range, no balancing needed
- return null;
- }
-
- // Determine source volume: highest utilization volume (if above threshold)
- final VolumeFixedUsage src = highestUsage;
-
- // Find destination volume: lowest utilization volume that has enough space
- // Prefer volumes below threshold, but accept any volume with lower utilization than source
- for (int i = 0; i < volumeUsages.size() - 1; i++) {
- final VolumeFixedUsage dstUsage = volumeUsages.get(i);
- final HddsVolume dst = dstUsage.getVolume();
-
- // Check if destination has enough space and has lower utilization than source
- if (dstUsage.getUtilization() < src.getUtilization() &&
- containerSize < dstUsage.computeUsableSpace()) {
- // Found dst, reserve space and return
- dst.incCommittedBytes(containerSize);
- LOG.debug("Chosen volume pair for disk balancing: source={} (utilization={}), " +
- "destination={} (utilization={})",
- src.getVolume().getStorageID(), src.getUtilization(),
- dst.getStorageID(), dstUsage.getUtilization());
- return Pair.of(src.getVolume(), dst);
- }
- LOG.debug("Destination volume {} does not have enough space, trying next volume.",
- dst.getStorageID());
- }
- LOG.debug("Failed to find appropriate destination volume.");
- return null;
- } finally {
- lock.unlock();
- }
- }
-
- /**
- * Logs all volume information for disk balancing investigation.
- *
- * @param volumeUsages List of volume usages (sorted by utilization ascending)
- * @param idealUsage Calculated ideal usage
- * @param thresholdPercentage Threshold percentage
- * @param lowerThreshold Lower threshold bound
- * @param upperThreshold Upper threshold bound
- * @param containerSize Container size to be moved
- * @param deltaMap Map of volume deltas
- */
- private void logVolumeBalancingState(List<VolumeFixedUsage> volumeUsages,
- double idealUsage, double thresholdPercentage, double lowerThreshold,
- double upperThreshold, long containerSize, Map<HddsVolume, Long> deltaMap) {
- LOG.debug("Disk balancing state - idealUsage={}, thresholdPercentage={}%, " +
- "thresholdRange=({}, {}), containerSize={}",
- String.format("%.10f", idealUsage), thresholdPercentage,
- String.format("%.10f", lowerThreshold), String.format("%.10f", upperThreshold),
- containerSize);
- for (int i = 0; i < volumeUsages.size(); i++) {
- VolumeFixedUsage vfu = volumeUsages.get(i);
- HddsVolume vol = vfu.getVolume();
- SpaceUsageSource.Fixed usage = vfu.getUsage();
- long usableSpace = vfu.computeUsableSpace();
- LOG.debug("Volume[{}] - disk={}, utilization={}, capacity={}, " +
- "effectiveUsed={}, available={}, usableSpace={}, committedBytes={}, delta={}",
- i, vol.getStorageID(), String.format("%.10f", vfu.getUtilization()),
- usage.getCapacity(), vfu.getEffectiveUsed(), usage.getAvailable(),
- usableSpace, vol.getCommittedBytes(), deltaMap.getOrDefault(vol, 0L));
- }
- }
-}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDefaultContainerChoosingPolicy.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDefaultContainerChoosingPolicy.java
index 67fa7dac39e..1c9e4ddeaa6 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDefaultContainerChoosingPolicy.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDefaultContainerChoosingPolicy.java
@@ -19,22 +19,28 @@
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
import static org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils.newContainerSet;
+import static org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerVolumeCalculation.getVolumeUsages;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.nio.file.Path;
import java.time.Duration;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Stream;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.fs.MockSpaceUsageCheckFactory;
@@ -45,12 +51,13 @@
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
+import org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerVolumeCalculation.VolumeFixedUsage;
+import org.apache.hadoop.ozone.container.diskbalancer.policy.ContainerCandidate;
import org.apache.hadoop.ozone.container.diskbalancer.policy.ContainerChoosingPolicy;
import org.apache.hadoop.ozone.container.diskbalancer.policy.DefaultContainerChoosingPolicy;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
@@ -58,11 +65,13 @@
import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
/**
- * Unit tests for the DefaultContainerChoosingPolicy.
+ * This class tests for volume and container selection in DefaultContainerChoosingPolicy.
*/
public class TestDefaultContainerChoosingPolicy {
@@ -72,171 +81,220 @@ public class TestDefaultContainerChoosingPolicy {
private static final OzoneConfiguration CONF = new OzoneConfiguration();
private static final long MB = 1024L * 1024L;
private static final long VOLUME_CAPACITY = 2500L * MB; // 2500MB
+ private static final long DEFAULT_CONTAINER_SIZE = 100L * MB; // 100MB
private static final double THRESHOLD = 10.0;
private ContainerChoosingPolicy policy;
- private OzoneContainer ozoneContainer;
private MutableVolumeSet volumeSet;
+ private String datanodeUuid;
+ private Map<HddsVolume, Long> deltaMap;
+ private OzoneContainer ozoneContainer;
private ContainerSet containerSet;
- private HddsVolume sourceVolume;
- private HddsVolume destVolume1;
- private HddsVolume destVolume2;
private Set<ContainerID> inProgressContainerIDs;
- private Map<HddsVolume, Long> deltaMap;
@BeforeEach
public void setup() throws Exception {
- policy = new DefaultContainerChoosingPolicy();
- setupVolumesAndContainer();
- inProgressContainerIDs = new HashSet<>();
+ datanodeUuid = UUID.randomUUID().toString();
+ policy = new DefaultContainerChoosingPolicy(new ReentrantLock());
deltaMap = new HashMap<>();
+ inProgressContainerIDs = new HashSet<>();
}
+ // ---------- Volume selection test structure ----------
+
/**
- * Sets up a scenario with three volumes and five containers on the source.
- * Volumes:
- * - Source Volume: 80% used (2000MB)
- * - Destination Volume 1: 10% used (250MB)
- * - Destination Volume 2: 50% used (1250MB)
- * Containers on Source (Total 2000MB):
- * - C1: 500MB, C2: 450MB, C3: 200MB, C4: 350MB, C5: 500MB
+ * Test case data structure for volume configuration.
*/
- private void setupVolumesAndContainer() throws IOException {
- // Create volumes with specific utilization
- sourceVolume = createVolume("source-volume", 0.80);
- destVolume1 = createVolume("dest-volume1", 0.10);
- destVolume2 = createVolume("dest-volume2", 0.50);
-
- CONF.set(HDDS_DATANODE_DIR_KEY, baseDir.resolve("defaultVolume").toString());
-
- // Create and spy on the volume set
- String datanodeUuid = UUID.randomUUID().toString();
- volumeSet = spy(new MutableVolumeSet(datanodeUuid, CONF, null,
- StorageVolume.VolumeType.DATA_VOLUME, null));
- when(volumeSet.getVolumesList())
- .thenReturn(Arrays.asList(sourceVolume, destVolume1, destVolume2));
-
- // Create five containers on the source volume
- containerSet = newContainerSet();
- Map<Long, Long> containerSizes = new HashMap<>();
- containerSizes.put(1L, 500L * MB);
- containerSizes.put(2L, 450L * MB);
- containerSizes.put(3L, 200L * MB);
- containerSizes.put(4L, 350L * MB);
- containerSizes.put(5L, 500L * MB);
+ public static class VolumeTestConfig {
+ private final String name;
+ private final double utilization;
+ private final Long customCapacity; // null means use default VOLUME_CAPACITY
- for (Map.Entry<Long, Long> entry : containerSizes.entrySet()) {
- createContainer(entry.getKey(), entry.getValue(), sourceVolume);
+ public VolumeTestConfig(String name, double utilization) {
+ this.name = name;
+ this.utilization = utilization;
+ this.customCapacity = null;
}
- // Mock OzoneContainer to return our container set
- ozoneContainer = mock(OzoneContainer.class);
- ContainerController controller = new ContainerController(containerSet, null);
- when(ozoneContainer.getController()).thenReturn(controller);
+ public VolumeTestConfig(String name, double utilization, long customCapacity) {
+ this.name = name;
+ this.utilization = utilization;
+ this.customCapacity = customCapacity;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public double getUtilization() {
+ return utilization;
+ }
+
+ public Long getCustomCapacity() {
+ return customCapacity;
+ }
}
/**
- * Create volumes with specific utilization.
+ * Test scenario for chooseVolumesAndContainer.
*/
- private HddsVolume createVolume(String dir, double utilization)
+ public static class TestScenario {
+ private final String name;
+ private final List<VolumeTestConfig> volumes;
+ private final double thresholdPercentage;
+ private final long containerSize;
+ private final boolean shouldFindPair;
+ private final String expectedSourceDisk;
+ private final String expectedDestinationDisk;
+ private final Integer expectedSourceIndex;
+ private final Integer expectedDestinationIndex;
+
+ // Container selection: when non-null, create these on source instead of single container
+ private final Map<Long, Long> sourceContainers;
+
+ // When true, block lowest-util volume (simulate no space)
+ private final boolean blockDestination;
+
+ // Container IDs to add to inProgressContainerIDs before the call
+ private final List<Long> inProgressIds;
+
+ // When shouldFindPair, expected container ID (null = any)
+ private final Long expectedContainerId;
+
+ @SuppressWarnings("checkstyle:parameternumber")
+ public TestScenario(String name, List<VolumeTestConfig> volumes, double thresholdPercentage,
+ long containerSize, boolean shouldFindPair, String expectedSourceDisk, String expectedDestinationDisk,
+ Integer expectedSourceIndex, Integer expectedDestinationIndex) {
+ this(name, volumes, thresholdPercentage, containerSize, shouldFindPair,
+ expectedSourceDisk, expectedDestinationDisk, expectedSourceIndex, expectedDestinationIndex,
+ null, false, null, null);
+ }
+
+ @SuppressWarnings("checkstyle:parameternumber")
+ public TestScenario(String name, List<VolumeTestConfig> volumes, double thresholdPercentage,
+ long containerSize, boolean shouldFindPair, String expectedSourceDisk, String expectedDestinationDisk,
+ Integer expectedSourceIndex, Integer expectedDestinationIndex,
+ Map<Long, Long> sourceContainers, boolean blockDestination, List<Long> inProgressIds,
+ Long expectedContainerId) {
+ this.name = name;
+ this.volumes = volumes;
+ this.thresholdPercentage = thresholdPercentage;
+ this.containerSize = containerSize;
+ this.shouldFindPair = shouldFindPair;
+ this.expectedSourceDisk = expectedSourceDisk;
+ this.expectedDestinationDisk = expectedDestinationDisk;
+ this.expectedSourceIndex = expectedSourceIndex;
+ this.expectedDestinationIndex = expectedDestinationIndex;
+ this.sourceContainers = sourceContainers;
+ this.blockDestination = blockDestination;
+ this.inProgressIds = inProgressIds;
+ this.expectedContainerId = expectedContainerId;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public List<VolumeTestConfig> getVolumes() {
+ return volumes;
+ }
+
+ public double getThresholdPercentage() {
+ return thresholdPercentage;
+ }
+
+ public long getContainerSize() {
+ return containerSize;
+ }
+
+ public boolean shouldFindPair() {
+ return shouldFindPair;
+ }
+
+ public String getExpectedSourceDisk() {
+ return expectedSourceDisk;
+ }
+
+ public String getExpectedDestinationDisk() {
+ return expectedDestinationDisk;
+ }
+
+ public Integer getExpectedSourceIndex() {
+ return expectedSourceIndex;
+ }
+
+ public Integer getExpectedDestinationIndex() {
+ return expectedDestinationIndex;
+ }
+
+ public Map<Long, Long> getSourceContainers() {
+ return sourceContainers;
+ }
+
+ public boolean isBlockDestination() {
+ return blockDestination;
+ }
+
+ public List<Long> getInProgressIds() {
+ return inProgressIds;
+ }
+
+ public Long getExpectedContainerId() {
+ return expectedContainerId;
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+ }
+
+ // ---------- Shared methods ----------
+
+ private HddsVolume createVolume(String name, double utilization, long capacity)
throws IOException {
- long usedSpace = (long) (VOLUME_CAPACITY * utilization);
- Path volumePath = baseDir.resolve(dir);
+ long usedSpace = (long) (capacity * utilization);
+ Path volumePath = baseDir.resolve(name);
+
+ // Create a configuration without reserved space to avoid capacity adjustments
+ OzoneConfiguration volumeConf = new OzoneConfiguration();
+ volumeConf.setFloat("hdds.datanode.dir.du.reserved.percent", 0.0f);
- SpaceUsageSource source = MockSpaceUsageSource.fixed(VOLUME_CAPACITY,
- VOLUME_CAPACITY - usedSpace);
+ SpaceUsageSource source = MockSpaceUsageSource.fixed(capacity,
+ capacity - usedSpace);
SpaceUsageCheckFactory factory = MockSpaceUsageCheckFactory.of(
source, Duration.ZERO, SpaceUsagePersistence.None.INSTANCE);
return new HddsVolume.Builder(volumePath.toString())
- .conf(CONF)
+ .conf(volumeConf)
.usageCheckFactory(factory)
.build();
}
/**
- * Create KeyValueContainers and add it to the containerSet.
+ * Creates volumes from test configuration.
+ *
+ * @param configs List of volume configurations
+ * @return List of created HddsVolumes
*/
- private void createContainer(long id, long size, HddsVolume vol)
+ private List<HddsVolume> createVolumes(List<VolumeTestConfig> configs)
throws IOException {
- createContainer(id, size, vol, containerSet);
- }
-
- @Test
- public void testContainerChosenSuccessfully() {
- // The policy should choose the first productive container for destVolume1.
- // Total Used: 2000MB + 250MB + 1250MB = 3500MB
- // Total Capacity: 3 * 2500MB = 7500MB
- // Ideal Usage: 3500 / 7500 = ~46.67%
- // Max Allowed Utilization: 46.67% + 10% (threshold) = ~56.67%
- //
- // Evaluation for destVolume1 (10% used / 250MB):
- // - C1 (500MB) is productive: (250+500)/2500 = 30% <= 56.67%
- // The policy iterates by container ID, so it will find and return C1.
-
- ContainerData chosenContainer = policy.chooseContainer(ozoneContainer,
- sourceVolume, destVolume1, inProgressContainerIDs, THRESHOLD, volumeSet, deltaMap);
-
- // first container should be chosen
- assertNotNull(chosenContainer);
- assertEquals(1L, chosenContainer.getContainerID());
- }
-
- @Test
- public void testContainerNotChosen() {
- // For destVolume2, no container move should be productive.
- // Max Allowed Utilization is ~56.67%.
- //
- // Evaluation for destVolume2 (50% used / 1250MB):
- // Max productive size = (0.5667 * 2500) - 1250 = 166.75MB
- // All containers on the source volume (smallest is 200MB) are larger
- // than 166.75MB. Therefore, no container should be chosen.
-
- ContainerData chosenContainer = policy.chooseContainer(ozoneContainer,
- sourceVolume, destVolume2, inProgressContainerIDs, THRESHOLD, volumeSet, deltaMap);
-
- // No containers should not be chosen
- assertNull(chosenContainer);
+ List<HddsVolume> volumes = new ArrayList<>();
+ for (VolumeTestConfig config : configs) {
+ long capacity = config.getCustomCapacity() != null ?
+ config.getCustomCapacity() : VOLUME_CAPACITY;
+ HddsVolume volume = createVolume(config.getName(), config.getUtilization(), capacity);
+ volumes.add(volume);
+ }
+ return volumes;
}
- @Test
- public void testSizeZeroContainersSkipped() throws IOException {
- // Create a new container set with containers that have size 0
- ContainerSet testContainerSet = newContainerSet();
-
- // Create containers with size 0 (should be skipped)
- createContainer(10L, 0L, sourceVolume, testContainerSet);
- createContainer(11L, 0L, sourceVolume, testContainerSet);
-
- // Create a container with non-zero size (should be chosen)
- createContainer(12L, 200L * MB, sourceVolume, testContainerSet);
-
- // Mock OzoneContainer to return our test container set
- OzoneContainer testOzoneContainer = mock(OzoneContainer.class);
- ContainerController testController = new ContainerController(testContainerSet, null);
- when(testOzoneContainer.getController()).thenReturn(testController);
-
- // The policy should skip containers 10 and 11 (size 0) and choose container 12
- ContainerData chosenContainer = policy.chooseContainer(testOzoneContainer,
- sourceVolume, destVolume1, inProgressContainerIDs, THRESHOLD, volumeSet, deltaMap);
-
- // Container 12 (non-zero size) should be chosen, skipping containers 10 and 11 (size 0)
- assertNotNull(chosenContainer);
- assertEquals(12L, chosenContainer.getContainerID());
- assertEquals(200L * MB, chosenContainer.getBytesUsed());
+ private void createContainer(long id, long usedBytes, HddsVolume vol) throws IOException {
+ createContainer(id, usedBytes, vol, containerSet);
}
- /**
- * Create KeyValueContainers and add it to the specified containerSet.
- * @param id container ID
- * @param usedBytes bytes used by the container (can be 0)
- * @param vol volume where container is located
- * @param targetContainerSet container set to add the container to
- */
- private void createContainer(long id, long usedBytes, HddsVolume vol,
- ContainerSet targetContainerSet) throws IOException {
- // Use maxSize as the container capacity (must be > 0)
- // If usedBytes is 0, we still need a valid maxSize for container creation
+ private void createContainer(long id, long usedBytes, HddsVolume vol, ContainerSet targetSet)
+ throws IOException {
long maxSize = usedBytes > 0 ? usedBytes : (long) CONF.getStorageSize(
ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
@@ -245,9 +303,385 @@ private void createContainer(long id, long usedBytes, HddsVolume vol,
UUID.randomUUID().toString(), UUID.randomUUID().toString());
containerData.setState(ContainerDataProto.State.CLOSED);
containerData.setVolume(vol);
- // Set the actual used bytes (can be 0)
containerData.getStatistics().setBlockBytesForTesting(usedBytes);
KeyValueContainer container = new KeyValueContainer(containerData, CONF);
- targetContainerSet.addContainer(container);
+ targetSet.addContainer(container);
+ }
+
+ private void setupVolumeSetAndContainers(List<HddsVolume> volumes,
+ HddsVolume sourceVolume, HddsVolume lowestVolume, TestScenario scenario) throws IOException {
+ OzoneConfiguration testConf = new OzoneConfiguration();
+ testConf.set(HDDS_DATANODE_DIR_KEY, baseDir.resolve("defaultVolume").toString());
+ volumeSet = new MutableVolumeSet(datanodeUuid, testConf, null,
+ StorageVolume.VolumeType.DATA_VOLUME, null);
+
+ // Use setVolumeMapForTesting to set only our test volumes
+ // This replaces the entire volumeMap, removing any default volumes from configuration
+ Map<String, StorageVolume> volumeMap = new HashMap<>();
+ for (HddsVolume volume : volumes) {
+ volumeMap.put(volume.getStorageDir().getAbsolutePath(), volume);
+ }
+ volumeSet.setVolumeMapForTesting(volumeMap);
+
+ containerSet = newContainerSet();
+ if (sourceVolume != null) {
+ Map<Long, Long> containerSizes = scenario.getSourceContainers();
+ if (containerSizes != null) {
+ for (Map.Entry<Long, Long> e : containerSizes.entrySet()) {
+ createContainer(e.getKey(), e.getValue(), sourceVolume);
+ }
+ } else {
+ createContainer(1L, scenario.getContainerSize(), sourceVolume);
+ }
+ }
+
+ if (scenario.isBlockDestination() && lowestVolume != null) {
+ lowestVolume.incCommittedBytes(lowestVolume.getCurrentUsage().getAvailable());
+ }
+ if (scenario.getInProgressIds() != null) {
+ for (Long id : scenario.getInProgressIds()) {
+ inProgressContainerIDs.add(ContainerID.valueOf(id));
+ }
+ }
+
+ ozoneContainer = mock(OzoneContainer.class);
+ ContainerController controller = new ContainerController(containerSet, null);
+ when(ozoneContainer.getController()).thenReturn(controller);
+ when(ozoneContainer.getContainerSet()).thenReturn(containerSet);
+ }
+
+ private MutableVolumeSet createVolumeSetForUsages(List<HddsVolume> volumes) throws IOException {
+ OzoneConfiguration testConf = new OzoneConfiguration();
+ testConf.set(HDDS_DATANODE_DIR_KEY, baseDir.resolve("defaultVolume").toString());
+ MutableVolumeSet vs = new MutableVolumeSet(datanodeUuid, testConf, null,
+ StorageVolume.VolumeType.DATA_VOLUME, null);
+ Map<String, StorageVolume> volumeMap = new HashMap<>();
+ for (HddsVolume volume : volumes) {
+ volumeMap.put(volume.getStorageDir().getAbsolutePath(), volume);
+ }
+ vs.setVolumeMapForTesting(volumeMap);
+ return vs;
+ }
+
+ /**
+ * Generic test method that can be reused for different scenarios.
+ *
+ * @param scenario Test scenario configuration
+ */
+ @ParameterizedTest(name = "{0}")
+ @MethodSource("testScenarios")
+ public void testChooseVolumesAndContainer(TestScenario scenario) throws IOException {
+ inProgressContainerIDs.clear();
+
+ List<HddsVolume> volumes = createVolumes(scenario.getVolumes());
+ List<VolumeFixedUsage> sortedUsages = getVolumeUsages(
+ createVolumeSetForUsages(volumes), deltaMap);
+ sortedUsages.sort(Comparator.comparingDouble(VolumeFixedUsage::getUtilization));
+ HddsVolume sourceVolume = scenario.shouldFindPair() || scenario.getSourceContainers() != null
+ ? sortedUsages.get(sortedUsages.size() - 1).getVolume() : null;
+ HddsVolume lowestVolume = sortedUsages.isEmpty() ? null : sortedUsages.get(0).getVolume();
+ setupVolumeSetAndContainers(volumes, sourceVolume, lowestVolume, scenario);
+
+ // Create a map of disk names to volumes for verification
+ Map<String, HddsVolume> diskNameToVolume = new HashMap<>();
+ for (int i = 0; i < scenario.getVolumes().size(); i++) {
+ diskNameToVolume.put(scenario.getVolumes().get(i).getName(), volumes.get(i));
+ }
+
+ // Get volume usages for verification
+ List<VolumeFixedUsage> volumeUsages = getVolumeUsages(volumeSet, deltaMap);
+
+ ContainerCandidate result = policy.chooseVolumesAndContainer(ozoneContainer,
+ volumeSet, deltaMap, inProgressContainerIDs, scenario.getThresholdPercentage());
+
+ if (scenario.shouldFindPair()) {
+ assertNotNull(result);
+ assertNotNull(result.getSourceVolume());
+ assertNotNull(result.getDestVolume());
+ assertNotNull(result.getContainerData());
+
+ // Verify source is the expected disk
+ if (scenario.getExpectedSourceDisk() != null) {
+ assertEquals(diskNameToVolume.get(scenario.getExpectedSourceDisk()), result.getSourceVolume());
+ }
+
+ // Verify destination is the expected disk (or one of the valid options)
+ if (scenario.getExpectedDestinationDisk() != null) {
+ assertEquals(diskNameToVolume.get(scenario.getExpectedDestinationDisk()), result.getDestVolume());
+ }
+ if (scenario.getExpectedContainerId() != null) {
+ assertEquals(scenario.getExpectedContainerId(), (Long) result.getContainerData().getContainerID());
+ }
+
+ // Filter volumeUsages to only include volumes from our test scenario
+ // This excludes any extra volumes that might be added from default configuration
+ List<VolumeFixedUsage> testVolumeUsages = new ArrayList<>();
+ for (VolumeFixedUsage usage : volumeUsages) {
+ if (diskNameToVolume.containsValue(usage.getVolume())) {
+ testVolumeUsages.add(usage);
+ }
+ }
+ // Sort by utilization to ensure consistent ordering
+ testVolumeUsages.sort(Comparator.comparingDouble(VolumeFixedUsage::getUtilization));
+
+ // Verify source and destination indices match expected values
+ // Since volumes in TestScenario are ordered from lowest to highest utilization,
+ // and testVolumeUsages are sorted ascending (lowest to highest),
+ // we can verify that the selected volumes are at the expected indices
+ int sourceIndex = -1;
+ int destIndex = -1;
+ for (int i = 0; i < testVolumeUsages.size(); i++) {
+ if (testVolumeUsages.get(i).getVolume().equals(result.getSourceVolume())) {
+ sourceIndex = i;
+ }
+ if (testVolumeUsages.get(i).getVolume().equals(result.getDestVolume())) {
+ destIndex = i;
+ }
+ }
+ assertTrue(sourceIndex >= 0);
+ assertTrue(destIndex >= 0);
+
+ // Verify source is at the expected index (should be the highest utilization)
+ if (scenario.getExpectedSourceIndex() != null) {
+ assertEquals(scenario.getExpectedSourceIndex(), sourceIndex);
+ }
+ if (scenario.getExpectedDestinationIndex() != null) {
+ assertEquals(scenario.getExpectedDestinationIndex(), destIndex);
+
+ // Verify that destination has lower utilization than source
+ double destUtilization = testVolumeUsages.get(destIndex).getUtilization();
+ double sourceUtilization = testVolumeUsages.get(sourceIndex).getUtilization();
+ assertTrue(destUtilization < sourceUtilization);
+
+ // Verify that no volume with lower index (lower utilization) has sufficient space
+ // If a volume at a lower index had sufficient space, it would have been chosen instead
+ for (int i = 0; i < destIndex; i++) {
+ assertTrue(scenario.getContainerSize() >=
+ testVolumeUsages.get(i).computeUsableSpace());
+ }
+ }
+ } else {
+ assertNull(result);
+ }
+ }
+
+ private static Map<Long, Long> mapOf(long... kvPairs) {
+ Map<Long, Long> m = new HashMap<>();
+ for (int i = 0; i < kvPairs.length; i += 2) {
+ m.put(kvPairs[i], kvPairs[i + 1]);
+ }
+ return m;
+ }
+
+ /**
+ * Provides test scenarios for parameterized testing.
+ */
+ @SuppressWarnings("checkstyle:methodlength")
+ public static Stream<Arguments> testScenarios() {
+ return Stream.of(
+ // <---------------------Volume selection Scenarios--------------------->
+
+ // Scenario 1: One volume beyond threshold, no volumes under threshold
+ // Disk1: 30%, Disk2: 30.1%, Disk3: 40%, Threshold: 5%
+ // Ideal: 33.37%, Range: (28.37%, 38.37%), Out of range: Disk3
+ // Expected source: Disk3 (highest) at index 2, Expected destination: Disk1 (lowest) at index 0
+ Arguments.arguments(new TestScenario(
+ "OneVolumeBeyondThresholdNoVolumesUnderThreshold",
+ Arrays.asList(
+ new VolumeTestConfig("disk1", 0.30),
+ new VolumeTestConfig("disk2", 0.301),
+ new VolumeTestConfig("disk3", 0.40)
+ ), 5.0, DEFAULT_CONTAINER_SIZE, true, "disk3", "disk1", 2, 0)),
+
+ // Scenario 2: Volumes both above and below threshold
+ // Disk1: 90%, Disk2: 85%, Disk3: 15%, Threshold: 10%
+ // Ideal: 63.33%, Range: (53.33%, 73.33%), Out of range: Disk1, Disk2, Disk3
+ // Expected source: Disk1 (highest) at index 2, Expected destination: Disk3 (lowest) at index 0
+ Arguments.arguments(new TestScenario(
+ "VolumesAboveAndBelowThreshold",
+ Arrays.asList(
+ new VolumeTestConfig("disk3", 0.15),
+ new VolumeTestConfig("disk2", 0.85),
+ new VolumeTestConfig("disk1", 0.90)
+ ), 10.0, DEFAULT_CONTAINER_SIZE, true, "disk1", "disk3", 2, 0)),
+
+ // Scenario 3: All volumes within threshold
+ // Disk1: 30%, Disk2: 30%, Disk3: 33%, Threshold: 10%
+ // Ideal: 31%, Range: (21%, 41%), Out of range: None
+ // Expected source: None, Expected destination: None
+ Arguments.arguments(new TestScenario(
+ "AllVolumesWithinThreshold",
+ Arrays.asList(
+ new VolumeTestConfig("disk1", 0.30),
+ new VolumeTestConfig("disk2", 0.301),
+ new VolumeTestConfig("disk3", 0.33)
+ ), 10.0, DEFAULT_CONTAINER_SIZE, false, null, null, null, null)),
+
+ // Scenario 4: One volume under threshold, no volumes above threshold
+ // Disk1: 30%, Disk2: 30.1%, Disk3: 20%
+ // Ideal: 26.70%, Range: (21.70%, 31.70%), Out of range: Disk3
+ // Expected source: Disk2 (highest) at index 2, Expected destination: Disk3 (lowest) at index 0
+ Arguments.arguments(new TestScenario(
+ "OneVolumeUnderThresholdNoVolumesAbove",
+ Arrays.asList(
+ new VolumeTestConfig("disk3", 0.20),
+ new VolumeTestConfig("disk1", 0.30),
+ new VolumeTestConfig("disk2", 0.301)
+ ), 5.0, DEFAULT_CONTAINER_SIZE, true, "disk2", "disk3", 2, 0)),
+
+ // Scenario 5: Extreme imbalance - one very high, others very low
+ // Disk1: 95%, Disk2: 5%, Disk3: 5.1%, Threshold: 10%
+ // Ideal: 35.03%, Range: (25.03%, 45.03%), Out of range: Disk1, Disk2, Disk3
+ // Expected source: Disk1 (highest) at index 2, Expected destination: Disk2 (lowest) at index 0
+ Arguments.arguments(new TestScenario(
+ "ExtremeImbalance",
+ Arrays.asList(
+ new VolumeTestConfig("disk2", 0.05),
+ new VolumeTestConfig("disk3", 0.051),
+ new VolumeTestConfig("disk1", 0.95)
+ ), 10.0, DEFAULT_CONTAINER_SIZE, true, "disk1", "disk2", 2, 0)),
+
+ // Scenario 6: Multiple volumes above threshold, one below
+ // Disk1: 80%, Disk2: 75%, Disk3: 20%, Threshold: 10%
+ // Ideal: 58.33%, Range: (48.33%, 68.33%), Out of range: Disk1, Disk2, Disk3
+ // Expected source: Disk1 (highest) at index 2, Expected destination: Disk3 (lowest) at index 0
+ Arguments.arguments(new TestScenario(
+ "MultipleVolumesAboveOneBelow",
+ Arrays.asList(
+ new VolumeTestConfig("disk3", 0.20),
+ new VolumeTestConfig("disk2", 0.75),
+ new VolumeTestConfig("disk1", 0.80)
+ ), 10.0, DEFAULT_CONTAINER_SIZE, true, "disk1", "disk3", 2, 0)),
+
+ // Scenario 7: Edge case - volumes at threshold boundaries
+ // Disk1: 50%, Disk2: 40%, Disk3: 60%, Threshold: 10%
+ // Ideal: 50%, Range: (40%, 60%), Out of range: Disk3 (at upper), Disk2 (at lower)
+ // Expected source: Disk3 (highest) at index 2, Expected destination: Disk2 (lowest) at index 0
+ Arguments.arguments(new TestScenario(
+ "VolumesAtThresholdBoundaries",
+ Arrays.asList(
+ new VolumeTestConfig("disk2", 0.40),
+ new VolumeTestConfig("disk1", 0.50),
+ new VolumeTestConfig("disk3", 0.60)
+ ), 10.0, DEFAULT_CONTAINER_SIZE, true, "disk3", "disk2", 2, 0)),
+
+ // Scenario 8: Small threshold with moderate imbalance
+ // Disk1: 35%, Disk2: 30%, Disk3: 30.1%, Threshold: 2%
+ // Ideal: 31.70%, Range: (29.70%, 33.70%), Out of range: Disk1
+ // Container size must be small enough to fit on disk2 without exceeding threshold
+ // disk2 after: (750MB + 50MB) / 2500MB = 32% < 33.7% threshold ✓
+ // Expected source: Disk1 (highest) at index 2, Expected destination: Disk2 (lowest) at index 0
+ Arguments.arguments(new TestScenario(
+ "SmallThresholdModerateImbalance",
+ Arrays.asList(
+ new VolumeTestConfig("disk2", 0.30),
+ new VolumeTestConfig("disk3", 0.301),
+ new VolumeTestConfig("disk1", 0.35)
+ ), 2.0, 50L * MB, true, "disk1", "disk2", 2, 0)),
+
+ // Scenario 9: Best destination has low utilization but insufficient space
+ // Disk1: 90% (Source)
+ // Disk2: 10% (Best Util, but small capacity 500MB) -> REJECT (insufficient space)
+ // Disk3: 20% (Second Best Util, plenty of space) -> ACCEPT
+ // Disk2: capacity=500MB, 10% used=50MB, available=450MB
+ // Container size 500MB > 450MB available, so disk2 is rejected
+ // Disk3: capacity=2500MB, 20% used=500MB, available=2000MB
+ // Container size 500MB < 2000MB available, so disk3 is accepted
+ Arguments.arguments(new TestScenario(
+ "BestDestInsufficientSpace",
+ Arrays.asList(
+ new VolumeTestConfig("disk2", 0.10, 500L * MB),
+ new VolumeTestConfig("disk3", 0.20),
+ new VolumeTestConfig("disk1", 0.90)
+ ), 10.0, 500L * MB, true, "disk1", "disk3", 2, 1)),
+
+ // Scenario 10: Volumes just inside threshold boundaries
+ // Disk1: 40.01%, Disk2: 59.99%, Threshold: 10%
+ // Ideal: 50%, Range: (40%, 60%), Out of range: None (both are just inside)
+ // Expected: No pair should be found as all volumes are within threshold
+ Arguments.arguments(new TestScenario(
+ "VolumesJustInsideThresholdBoundaries",
+ Arrays.asList(
+ new VolumeTestConfig("disk1", 0.4001),
+ new VolumeTestConfig("disk2", 0.5999)
+ ), 10.0, DEFAULT_CONTAINER_SIZE, false, null, null, null, null)),
+
+ // Scenario 10b: Volumes just outside threshold boundaries
+ // Disk1: 39.99%, Disk2: 60.01%, Threshold: 10%
+ // Ideal: 50%, Range: (40%, 60%), Out of range: Disk1 (just below lower), Disk2 (just above upper)
+ // Expected: Pair should be found - Disk2 as source at index 1, Disk1 as destination at index 0
+ Arguments.arguments(new TestScenario(
+ "VolumesJustOutsideThresholdBoundaries",
+ Arrays.asList(
+ new VolumeTestConfig("disk1", 0.3999),
+ new VolumeTestConfig("disk2", 0.6001)
+ ), 10.0, DEFAULT_CONTAINER_SIZE, true, "disk2", "disk1", 1, 0)),
+
+ // Scenario 11: No volumes have enough free space
+ // Disk1: 90% (Source, highest)
+ // Disk2: 10% (Destination candidate, but insufficient space)
+ // Disk3: 20% (Destination candidate, but insufficient space)
+ // Disk2: capacity=500MB, 10% used=50MB, available=450MB
+ // Disk3: capacity=500MB, 20% used=100MB, available=400MB
+ // Container size 500MB > available space on both disk2 and disk3, so both are rejected
+ // No valid destination found
+ Arguments.arguments(new TestScenario(
+ "NoVolumesHaveEnoughFreeSpace",
+ Arrays.asList(
+ new VolumeTestConfig("disk2", 0.10, 500L * MB),
+ new VolumeTestConfig("disk3", 0.20, 500L * MB),
+ new VolumeTestConfig("disk1", 0.90)
+ ), 10.0, 500L * MB, false, null, null, null, null)),
+
+ // Scenario 12: Only one volume
+ // Disk1: 80%
+ // Cannot balance with only one volume
+ // Expected: No pair should be found
+ Arguments.arguments(new TestScenario(
+ "OnlyOneVolume",
+ Arrays.asList(new VolumeTestConfig("disk1", 0.80)),
+ 10.0, DEFAULT_CONTAINER_SIZE, false, null, null, null, null)),
+
+ // Scenario 13: Zero volumes (empty volume set)
+ // No volumes at all
+ // Expected: No pair should be found
+ Arguments.arguments(new TestScenario(
+ "ZeroVolumes",
+ Arrays.asList(), 10.0, DEFAULT_CONTAINER_SIZE, false, null, null, null, null)),
+
+ // <---------------------Container selection Scenarios--------------------->
+
+ // Container selection: multiple containers on source, first valid chosen
+ Arguments.arguments(new TestScenario(
+ "ContainerChosenSuccessfully",
+ Arrays.asList(
+ new VolumeTestConfig("dest-volume1", 0.10),
+ new VolumeTestConfig("dest-volume2", 0.50),
+ new VolumeTestConfig("source-volume", 0.80)
+ ), THRESHOLD, DEFAULT_CONTAINER_SIZE, true, "source-volume", "dest-volume1", 2, 0,
+ mapOf(1L, 500L * MB, 2L, 450L * MB, 3L, 200L * MB, 4L, 350L * MB, 5L, 500L * MB),
+ false, null, 1L)),
+
+ // Container selection: dest blocked, small containers in-progress, no fit
+ Arguments.arguments(new TestScenario(
+ "ContainerNotChosen",
+ Arrays.asList(
+ new VolumeTestConfig("dest-volume1", 0.10),
+ new VolumeTestConfig("dest-volume2", 0.50),
+ new VolumeTestConfig("source-volume", 0.80)
+ ), THRESHOLD, DEFAULT_CONTAINER_SIZE, false, null, null, null, null,
+ mapOf(1L, 500L * MB, 2L, 450L * MB, 3L, 200L * MB, 4L, 350L * MB, 5L, 500L * MB),
+ true, Arrays.asList(3L, 4L), null)),
+
+ // Container selection: size-zero containers skipped, first non-zero chosen
+ Arguments.arguments(new TestScenario(
+ "SizeZeroContainersSkipped",
+ Arrays.asList(
+ new VolumeTestConfig("dest-volume1", 0.10),
+ new VolumeTestConfig("dest-volume2", 0.50),
+ new VolumeTestConfig("source-volume", 0.80)
+ ), THRESHOLD, DEFAULT_CONTAINER_SIZE, true, "source-volume", "dest-volume1", 2, 0,
+ mapOf(10L, 0L, 11L, 0L, 12L, 200L * MB), false, null, 12L))
+ );
}
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDefaultVolumeChoosingPolicy.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDefaultVolumeChoosingPolicy.java
deleted file mode 100644
index ff47a8281ec..00000000000
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDefaultVolumeChoosingPolicy.java
+++ /dev/null
@@ -1,651 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.diskbalancer;
-
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
-import static org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerVolumeCalculation.getVolumeUsages;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import java.io.IOException;
-import java.nio.file.Path;
-import java.time.Duration;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.stream.Stream;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.fs.MockSpaceUsageCheckFactory;
-import org.apache.hadoop.hdds.fs.MockSpaceUsageSource;
-import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory;
-import org.apache.hadoop.hdds.fs.SpaceUsagePersistence;
-import org.apache.hadoop.hdds.fs.SpaceUsageSource;
-import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
-import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
-import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
-import org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerVolumeCalculation.VolumeFixedUsage;
-import org.apache.hadoop.ozone.container.diskbalancer.policy.DefaultVolumeChoosingPolicy;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.io.TempDir;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.Arguments;
-import org.junit.jupiter.params.provider.MethodSource;
-
-/**
- * Unit tests for DefaultVolumeChoosingPolicy.
- */
-public class TestDefaultVolumeChoosingPolicy {
-
- @TempDir
- private Path baseDir;
-
- private static final long MB = 1024L * 1024L;
- private static final long VOLUME_CAPACITY = 2500L * MB; // 2500MB - same for all volumes
- private static final long DEFAULT_CONTAINER_SIZE = 100L * MB; // 100MB
- private DefaultVolumeChoosingPolicy policy;
- private MutableVolumeSet volumeSet;
- private String datanodeUuid;
- private Map<HddsVolume, Long> deltaMap;
-
- @BeforeEach
- public void setup() {
- datanodeUuid = UUID.randomUUID().toString();
- policy = new DefaultVolumeChoosingPolicy(new ReentrantLock());
- deltaMap = new HashMap<>();
- }
-
- /**
- * Test case data structure for volume configuration.
- */
- public static class VolumeTestConfig {
- private final String name;
- private final double utilization;
- private final Long customCapacity; // null means use default VOLUME_CAPACITY
-
- public VolumeTestConfig(String name, double utilization) {
- this.name = name;
- this.utilization = utilization;
- this.customCapacity = null;
- }
-
- public VolumeTestConfig(String name, double utilization, long customCapacity) {
- this.name = name;
- this.utilization = utilization;
- this.customCapacity = customCapacity;
- }
-
- public String getName() {
- return name;
- }
-
- public double getUtilization() {
- return utilization;
- }
-
- public Long getCustomCapacity() {
- return customCapacity;
- }
- }
-
- /**
- * Test scenario configuration.
- */
- public static class TestScenario {
- private final String name;
- private final List<VolumeTestConfig> volumes;
- private final double thresholdPercentage;
- private final long containerSize;
- private final boolean shouldFindPair;
- private final String expectedSourceDisk;
- private final String expectedDestinationDisk;
- private final Integer expectedSourceIndex;
- private final Integer expectedDestinationIndex;
-
- @SuppressWarnings("checkstyle:parameternumber")
- public TestScenario(String name, List<VolumeTestConfig> volumes, double thresholdPercentage,
- long containerSize, boolean shouldFindPair, String expectedSourceDisk, String expectedDestinationDisk,
- Integer expectedSourceIndex, Integer expectedDestinationIndex) {
- this.name = name;
- this.volumes = volumes;
- this.thresholdPercentage = thresholdPercentage;
- this.containerSize = containerSize;
- this.shouldFindPair = shouldFindPair;
- this.expectedSourceDisk = expectedSourceDisk;
- this.expectedDestinationDisk = expectedDestinationDisk;
- this.expectedSourceIndex = expectedSourceIndex;
- this.expectedDestinationIndex = expectedDestinationIndex;
- }
-
- public String getName() {
- return name;
- }
-
- public List<VolumeTestConfig> getVolumes() {
- return volumes;
- }
-
- public double getThresholdPercentage() {
- return thresholdPercentage;
- }
-
- public long getContainerSize() {
- return containerSize;
- }
-
- public boolean shouldFindPair() {
- return shouldFindPair;
- }
-
- public String getExpectedSourceDisk() {
- return expectedSourceDisk;
- }
-
- public String getExpectedDestinationDisk() {
- return expectedDestinationDisk;
- }
-
- public Integer getExpectedSourceIndex() {
- return expectedSourceIndex;
- }
-
- public Integer getExpectedDestinationIndex() {
- return expectedDestinationIndex;
- }
-
- @Override
- public String toString() {
- return name;
- }
- }
-
- /**
- * Creates a volume with specific utilization and capacity.
- *
- * @param name Volume name/identifier
- * @param utilization Utilization as a double (0.0 to 1.0, representing percentage)
- * @param capacity capacity for the volume
- * @return HddsVolume with specified utilization and capacity
- */
- private HddsVolume createVolume(String name, double utilization, long capacity)
- throws IOException {
- long usedSpace = (long) (capacity * utilization);
- Path volumePath = baseDir.resolve(name);
-
- // Create a configuration without reserved space to avoid capacity adjustments
- OzoneConfiguration volumeConf = new OzoneConfiguration();
- volumeConf.setFloat("hdds.datanode.dir.du.reserved.percent", 0.0f);
-
- SpaceUsageSource source = MockSpaceUsageSource.fixed(capacity,
- capacity - usedSpace);
- SpaceUsageCheckFactory factory = MockSpaceUsageCheckFactory.of(
- source, Duration.ZERO, SpaceUsagePersistence.None.INSTANCE);
-
- HddsVolume volume = new HddsVolume.Builder(volumePath.toString())
- .conf(volumeConf)
- .usageCheckFactory(factory)
- .build();
- return volume;
- }
-
- /**
- * Creates volumes from test configuration.
- *
- * @param configs List of volume configurations
- * @return List of created HddsVolumes
- */
- private List<HddsVolume> createVolumes(List<VolumeTestConfig> configs)
- throws IOException {
- List<HddsVolume> volumes = new ArrayList<>();
- for (VolumeTestConfig config : configs) {
- long capacity = config.getCustomCapacity() != null ?
- config.getCustomCapacity() : VOLUME_CAPACITY;
- HddsVolume volume = createVolume(config.getName(), config.getUtilization(), capacity);
- volumes.add(volume);
- }
- return volumes;
- }
-
- /**
- * Sets up volume set with given volumes.
- *
- * @param volumes List of volumes to add to volume set
- */
- private void setupVolumeSet(List<HddsVolume> volumes) throws IOException {
- // Use a clean configuration to avoid loading default volumes
- OzoneConfiguration testConf = new OzoneConfiguration();
- testConf.set(HDDS_DATANODE_DIR_KEY, baseDir.resolve("defaultVolume").toString());
- volumeSet = new MutableVolumeSet(datanodeUuid, testConf, null,
- StorageVolume.VolumeType.DATA_VOLUME, null);
-
- // Use setVolumeMapForTesting to set only our test volumes
- // This replaces the entire volumeMap, removing any default volumes from configuration
- Map<String, StorageVolume> volumeMap = new HashMap<>();
- for (HddsVolume volume : volumes) {
- volumeMap.put(volume.getStorageDir().getAbsolutePath(), volume);
- }
- volumeSet.setVolumeMapForTesting(volumeMap);
- }
-
- /**
- * Generic test method that can be reused for different scenarios.
- *
- * @param scenario Test scenario configuration
- */
- @ParameterizedTest(name = "{0}")
- @MethodSource("testScenarios")
- public void testVolumeChoosingPolicy(TestScenario scenario)
- throws IOException {
- // Create volumes from configuration
- List<HddsVolume> volumes = createVolumes(scenario.getVolumes());
- setupVolumeSet(volumes);
-
- // Create a map of disk names to volumes for verification
- Map<String, HddsVolume> diskNameToVolume = new HashMap<>();
- for (int i = 0; i < scenario.getVolumes().size(); i++) {
- VolumeTestConfig config = scenario.getVolumes().get(i);
- diskNameToVolume.put(config.getName(), volumes.get(i));
- }
-
- // Get volume usages for verification
- List<VolumeFixedUsage> volumeUsages = getVolumeUsages(volumeSet, deltaMap);
-
- // Try to find a valid source-destination pair
- Pair<HddsVolume, HddsVolume> result = policy.chooseVolume(volumeSet,
- scenario.getThresholdPercentage(), deltaMap, scenario.getContainerSize());
-
- if (scenario.shouldFindPair()) {
- assertNotNull(result);
- assertNotNull(result.getLeft());
- assertNotNull(result.getRight());
-
- // Verify source is the expected disk
- if (scenario.getExpectedSourceDisk() != null) {
- HddsVolume expectedSource = diskNameToVolume.get(scenario.getExpectedSourceDisk());
- assertNotNull(expectedSource);
- assertEquals(expectedSource, result.getLeft());
- }
-
- // Verify destination is the expected disk (or one of the valid options)
- if (scenario.getExpectedDestinationDisk() != null) {
- HddsVolume expectedDest = diskNameToVolume.get(scenario.getExpectedDestinationDisk());
- assertNotNull(expectedDest);
- assertEquals(expectedDest, result.getRight());
- }
-
- // Filter volumeUsages to only include volumes from our test scenario
- // This excludes any extra volumes that might be added from default configuration
- List<VolumeFixedUsage> testVolumeUsages = new ArrayList<>();
- for (VolumeFixedUsage usage : volumeUsages) {
- if (diskNameToVolume.containsValue(usage.getVolume())) {
- testVolumeUsages.add(usage);
- }
- }
- // Sort by utilization to ensure consistent ordering
- testVolumeUsages.sort(Comparator.comparingDouble(VolumeFixedUsage::getUtilization));
-
- // Verify source and destination indices match expected values
- // Since volumes in TestScenario are ordered from lowest to highest utilization,
- // and testVolumeUsages are sorted ascending (lowest to highest),
- // we can verify that the selected volumes are at the expected indices
- int sourceIndex = -1;
- int destIndex = -1;
- for (int i = 0; i < testVolumeUsages.size(); i++) {
- if (testVolumeUsages.get(i).getVolume().equals(result.getLeft())) {
- sourceIndex = i;
- }
- if (testVolumeUsages.get(i).getVolume().equals(result.getRight())) {
- destIndex = i;
- }
- }
- assertTrue(sourceIndex >= 0);
- assertTrue(destIndex >= 0);
-
- // Verify source is at the expected index (should be the highest utilization)
- if (scenario.getExpectedSourceIndex() != null) {
- assertEquals(scenario.getExpectedSourceIndex(), sourceIndex);
-
- // Verify that source has the highest utilization
- double sourceUtilization = testVolumeUsages.get(sourceIndex).getUtilization();
- for (int i = 0; i < testVolumeUsages.size(); i++) {
- if (i != sourceIndex) {
- double otherUtilization = testVolumeUsages.get(i).getUtilization();
- assertTrue(sourceUtilization >= otherUtilization);
- }
- }
- }
-
- // Verify destination is at the expected index (should be the lowest utilization among valid destinations)
- if (scenario.getExpectedDestinationIndex() != null) {
- assertEquals(scenario.getExpectedDestinationIndex(), destIndex);
-
- // Verify that destination has lower utilization than source
- double destUtilization = testVolumeUsages.get(destIndex).getUtilization();
- double sourceUtilization = testVolumeUsages.get(sourceIndex).getUtilization();
- assertTrue(destUtilization < sourceUtilization);
-
- // Verify that no volume with lower index (lower utilization) has sufficient space
- // If a volume at a lower index had sufficient space, it would have been chosen instead
- for (int i = 0; i < destIndex; i++) {
- VolumeFixedUsage lowerUtilUsage = testVolumeUsages.get(i);
- long usableSpace = lowerUtilUsage.computeUsableSpace();
- // Policy checks: containerSize < computeUsableSpace() to see if there's enough space
- // So if containerSize >= usableSpace, there's NOT enough space (correct - volume wasn't chosen)
- assertTrue(scenario.getContainerSize() >= usableSpace);
- }
- }
- } else {
- assertNull(result);
- }
- }
-
- /**
- * Provides test scenarios for parameterized testing.
- */
- @SuppressWarnings("checkstyle:methodlength")
- public static Stream<Arguments> testScenarios() {
- return Stream.of(
- // Scenario 1: One volume beyond threshold, no volumes under threshold
- // Disk1: 30%, Disk2: 30%, Disk3: 40%, Threshold: 5%
- // Ideal: 33.33%, Range: (28.33%, 38.33%), Out of range: Disk3
- // Expected source: Disk3 (highest) at index 2, Expected destination: Disk1 or Disk2 (lowest) at index 0 or 1
- Arguments.arguments(new TestScenario(
- "OneVolumeBeyondThresholdNoVolumesUnderThreshold",
- Arrays.asList(
- new VolumeTestConfig("disk1", 0.30), // Lowest utilization - index 0
- new VolumeTestConfig("disk2", 0.30), // Same as disk1 - index 1
- new VolumeTestConfig("disk3", 0.40) // Highest utilization - index 2
- ),
- 5.0,
- DEFAULT_CONTAINER_SIZE,
- true,
- "disk3", // Expected source (highest)
- null, // Destination can be disk1 or disk2 (both valid) so only check source
- 2, // Expected source index (highest utilization)
- null // Destination index can be 0 or 1 (both have same utilization)
- )),
-
- // Scenario 2: Volumes both above and below threshold
- // Disk1: 90%, Disk2: 85%, Disk3: 15%, Threshold: 10%
- // Ideal: 63.33%, Range: (53.33%, 73.33%), Out of range: Disk1, Disk2, Disk3
- // Expected source: Disk1 (highest) at index 2, Expected destination: Disk3 (lowest) at index 0
- Arguments.arguments(new TestScenario(
- "VolumesAboveAndBelowThreshold",
- Arrays.asList(
- new VolumeTestConfig("disk3", 0.15), // Lowest utilization - index 0
- new VolumeTestConfig("disk2", 0.85), // Middle utilization - index 1
- new VolumeTestConfig("disk1", 0.90) // Highest utilization - index 2
- ),
- 10.0,
- DEFAULT_CONTAINER_SIZE,
- true,
- "disk1", // Expected source (highest)
- "disk3", // Expected destination (lowest)
- 2, // Expected source index (highest utilization)
- 0 // Expected destination index (lowest utilization)
- )),
-
- // Scenario 3: All volumes within threshold
- // Disk1: 30%, Disk2: 30%, Disk3: 33%, Threshold: 10%
- // Ideal: 31%, Range: (21%, 41%), Out of range: None
- // Expected source: None, Expected destination: None
- Arguments.arguments(new TestScenario(
- "AllVolumesWithinThreshold",
- Arrays.asList(
- new VolumeTestConfig("disk1", 0.30), // Lowest utilization
- new VolumeTestConfig("disk2", 0.30), // Same as disk1
- new VolumeTestConfig("disk3", 0.33) // Highest utilization
- ),
- 10.0,
- DEFAULT_CONTAINER_SIZE,
- false,
- null, // No pair expected
- null,
- null, // No source index expected
- null // No destination index expected
- )),
-
- // Scenario 4: One volume under threshold, no volumes above threshold
- // Disk1: 30%, Disk2: 30%, Disk3: 20%
- // Ideal: 26.67%, Range: (21.67%, 31.67%), Out of range: Disk3
- // Expected source: Disk1 or Disk2 (highest) at index 1 or 2, Expected destination: Disk3 (lowest) at index 0
- Arguments.arguments(new TestScenario(
- "OneVolumeUnderThresholdNoVolumesAbove",
- Arrays.asList(
- new VolumeTestConfig("disk3", 0.20), // Lowest utilization - index 0
- new VolumeTestConfig("disk1", 0.30), // Middle utilization - index 1
- new VolumeTestConfig("disk2", 0.30) // Highest utilization (tied with disk1) - index 2
- ),
- 5.0,
- DEFAULT_CONTAINER_SIZE,
- true,
- null, // Source can be disk1 or disk2 (both valid, highest)
- "disk3", // Expected destination (lowest)
- null, // Source index can be 1 or 2 (both have same utilization)
- 0 // Expected destination index (lowest utilization)
- )),
-
- // Scenario 5: Extreme imbalance - one very high, others very low
- // Disk1: 95%, Disk2: 5%, Disk3: 5%, Threshold: 10%
- // Ideal: 35%, Range: (25%, 45%), Out of range: Disk1, Disk2, Disk3
- // Expected source: Disk1 (highest) at index 2, Expected destination: Disk2 or Disk3 (lowest) at index 0 or 1
- Arguments.arguments(new TestScenario(
- "ExtremeImbalance",
- Arrays.asList(
- new VolumeTestConfig("disk2", 0.05), // Lowest utilization - index 0
- new VolumeTestConfig("disk3", 0.05), // Same as disk2 - index 1
- new VolumeTestConfig("disk1", 0.95) // Highest utilization - index 2
- ),
- 10.0,
- DEFAULT_CONTAINER_SIZE,
- true,
- "disk1", // Expected source (highest)
- null, // Destination can be disk2 or disk3 (lowest)
- 2, // Expected source index (highest utilization)
- null // Destination index can be 0 or 1 (both have same utilization)
- )),
-
- // Scenario 6: Multiple volumes above threshold, one below
- // Disk1: 80%, Disk2: 75%, Disk3: 20%, Threshold: 10%
- // Ideal: 58.33%, Range: (48.33%, 68.33%), Out of range: Disk1, Disk2, Disk3
- // Expected source: Disk1 (highest) at index 2, Expected destination: Disk3 (lowest) at index 0
- Arguments.arguments(new TestScenario(
- "MultipleVolumesAboveOneBelow",
- Arrays.asList(
- new VolumeTestConfig("disk3", 0.20), // Lowest utilization - index 0
- new VolumeTestConfig("disk2", 0.75), // Middle utilization - index 1
- new VolumeTestConfig("disk1", 0.80) // Highest utilization - index 2
- ),
- 10.0,
- DEFAULT_CONTAINER_SIZE,
- true,
- "disk1", // Expected source (highest)
- "disk3", // Expected destination (lowest)
- 2, // Expected source index (highest utilization)
- 0 // Expected destination index (lowest utilization)
- )),
-
- // Scenario 7: Edge case - volumes at threshold boundaries
- // Disk1: 50%, Disk2: 40%, Disk3: 60%, Threshold: 10%
- // Ideal: 50%, Range: (40%, 60%), Out of range: Disk3 (at upper), Disk2 (at lower)
- // Expected source: Disk3 (highest) at index 2, Expected destination: Disk2 (lowest) at index 0
- Arguments.arguments(new TestScenario(
- "VolumesAtThresholdBoundaries",
- Arrays.asList(
- new VolumeTestConfig("disk2", 0.40), // Lowest utilization - index 0
- new VolumeTestConfig("disk1", 0.50), // Middle utilization - index 1
- new VolumeTestConfig("disk3", 0.60) // Highest utilization - index 2
- ),
- 10.0,
- DEFAULT_CONTAINER_SIZE,
- true,
- "disk3", // Expected source (highest)
- "disk2", // Expected destination (lowest)
- 2, // Expected source index (highest utilization)
- 0 // Expected destination index (lowest utilization)
- )),
-
- // Scenario 8: Small threshold with moderate imbalance
- // Disk1: 35%, Disk2: 30%, Disk3: 30%, Threshold: 2%
- // Ideal: 31.67%, Range: (29.67%, 33.67%), Out of range: Disk1
- // Expected source: Disk1 (highest) at index 2, Expected destination: Disk2 or Disk3 (lowest) at index 0 or 1
- Arguments.arguments(new TestScenario(
- "SmallThresholdModerateImbalance",
- Arrays.asList(
- new VolumeTestConfig("disk2", 0.30), // Lowest utilization - index 0
- new VolumeTestConfig("disk3", 0.30), // Same as disk2 - index 1
- new VolumeTestConfig("disk1", 0.35) // Highest utilization - index 2
- ),
- 2.0,
- DEFAULT_CONTAINER_SIZE,
- true,
- "disk1", // Expected source (highest)
- null, // Destination can be disk2 or disk3 (lowest)
- 2, // Expected source index (highest utilization)
- null // Destination index can be 0 or 1 (both have same utilization)
- )),
-
- // Scenario 9: Best destination has low utilization but insufficient space
- // Disk1: 90% (Source)
- // Disk2: 10% (Best Util, but small capacity 500MB) -> REJECT (insufficient space)
- // Disk3: 20% (Second Best Util, plenty of space) -> ACCEPT
- // Disk2: capacity=500MB, 10% used=50MB, available=450MB
- // Container size 500MB > 450MB available, so disk2 is rejected
- // Disk3: capacity=2500MB, 20% used=500MB, available=2000MB
- // Container size 500MB < 2000MB available, so disk3 is accepted
- Arguments.arguments(new TestScenario(
- "BestDestInsufficientSpace",
- Arrays.asList(
- new VolumeTestConfig("disk2", 0.10, 500L * MB), // Lowest utilization, small capacity - index 0
- new VolumeTestConfig("disk3", 0.20), // Second lowest - index 1
- new VolumeTestConfig("disk1", 0.90) // Highest utilization - index 2
- ),
- 10.0,
- 500L * MB, // Container size larger than disk2's available space
- true,
- "disk1", // Expected source (highest)
- "disk3", // Should skip disk2 and pick disk3
- 2, // Expected source index (highest utilization)
- 1 // Expected destination index (disk3 at index 1, since disk2 at index 0 doesn't have enough space)
- )),
-
- // Scenario 10: Volumes just inside threshold boundaries
- // Disk1: 40.01%, Disk2: 59.99%, Threshold: 10%
- // Ideal: 50%, Range: (40%, 60%), Out of range: None (both are just inside)
- // Expected: No pair should be found as all volumes are within threshold
- Arguments.arguments(new TestScenario(
- "VolumesJustInsideThresholdBoundaries",
- Arrays.asList(
- new VolumeTestConfig("disk1", 0.4001), // Lowest utilization - index 0
- new VolumeTestConfig("disk2", 0.5999) // Highest utilization - index 1
- ),
- 10.0,
- DEFAULT_CONTAINER_SIZE,
- false, // Should not find pair - both volumes are within threshold
- null,
- null,
- null, // No source index expected
- null // No destination index expected
- )),
-
- // Scenario 10b: Volumes just outside threshold boundaries
- // Disk1: 39.99%, Disk2: 60.01%, Threshold: 10%
- // Ideal: 50%, Range: (40%, 60%), Out of range: Disk1 (just below lower), Disk2 (just above upper)
- // Expected: Pair should be found - Disk2 as source at index 1, Disk1 as destination at index 0
- Arguments.arguments(new TestScenario(
- "VolumesJustOutsideThresholdBoundaries",
- Arrays.asList(
- new VolumeTestConfig("disk1", 0.3999), // Lowest utilization - index 0
- new VolumeTestConfig("disk2", 0.6001) // Highest utilization - index 1
- ),
- 10.0,
- DEFAULT_CONTAINER_SIZE,
- true, // Should find pair - volumes are outside threshold
- "disk2", // Expected source (highest utilization)
- "disk1", // Expected destination (lowest utilization)
- 1, // Expected source index (highest utilization)
- 0 // Expected destination index (lowest utilization)
- )),
-
- // Scenario 11: No volumes have enough free space
- // Disk1: 90% (Source, highest)
- // Disk2: 10% (Destination candidate, but insufficient space)
- // Disk3: 20% (Destination candidate, but insufficient space)
- // Disk2: capacity=500MB, 10% used=50MB, available=450MB
- // Disk3: capacity=500MB, 20% used=100MB, available=400MB
- // Container size 500MB > available space on both disk2 and disk3, so both are rejected
- // No valid destination found
- Arguments.arguments(new TestScenario(
- "NoVolumesHaveEnoughFreeSpace",
- Arrays.asList(
- new VolumeTestConfig("disk2", 0.10, 500L * MB), // Lowest utilization - index 0
- new VolumeTestConfig("disk3", 0.20, 500L * MB), // Second lowest - index 1
- new VolumeTestConfig("disk1", 0.90) // Highest utilization - index 2
- ),
- 10.0,
- 500L * MB, // Container size larger than both disk2's and disk3's available space
- false, // Should not find pair - no destination has enough space
- null,
- null,
- null, // No source index expected
- null // No destination index expected
- )),
-
- // Scenario 12: Only one volume
- // Disk1: 80%
- // Cannot balance with only one volume
- // Expected: No pair should be found
- Arguments.arguments(new TestScenario(
- "OnlyOneVolume",
- Arrays.asList(
- new VolumeTestConfig("disk1", 0.80) // Only volume
- ),
- 10.0,
- DEFAULT_CONTAINER_SIZE,
- false, // Should not find pair - need at least 2 volumes
- null,
- null,
- null, // No source index expected
- null // No destination index expected
- )),
-
- // Scenario 13: Zero volumes (empty volume set)
- // No volumes at all
- // Expected: No pair should be found
- Arguments.arguments(new TestScenario(
- "ZeroVolumes",
- Arrays.asList(), // Empty list
- 10.0,
- DEFAULT_CONTAINER_SIZE,
- false, // Should not find pair - no volumes available
- null,
- null,
- null, // No source index expected
- null // No destination index expected
- ))
- );
- }
-}
-
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java
index fc1980e579e..5a26a7b569e 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java
@@ -27,7 +27,6 @@
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyDouble;
-import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -39,7 +38,6 @@
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -56,10 +54,9 @@
import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerVolumeCalculation.VolumeFixedUsage;
+import org.apache.hadoop.ozone.container.diskbalancer.policy.ContainerCandidate;
import org.apache.hadoop.ozone.container.diskbalancer.policy.ContainerChoosingPolicy;
import org.apache.hadoop.ozone.container.diskbalancer.policy.DefaultContainerChoosingPolicy;
-import org.apache.hadoop.ozone.container.diskbalancer.policy.DefaultVolumeChoosingPolicy;
-import org.apache.hadoop.ozone.container.diskbalancer.policy.DiskBalancerVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.keyvalue.ContainerTestVersionInfo;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
@@ -210,10 +207,8 @@ public void testPolicyClassInitialization(ContainerTestVersionInfo versionInfo)
DiskBalancerServiceTestImpl svc =
getDiskBalancerService(containerSet, conf, keyValueHandler, null, 1);
- assertTrue(svc.getContainerChoosingPolicy()
+ assertTrue(svc.getVolumeContainerChoosingPolicy()
instanceof DefaultContainerChoosingPolicy);
- assertTrue(svc.getVolumeChoosingPolicy()
- instanceof DefaultVolumeChoosingPolicy);
}
private String generateVolumeLocation(String base, int volumeCount) {
@@ -321,10 +316,8 @@ public void testConcurrentTasksNotExceedThreadLimit() throws Exception {
false, DiskBalancerVersion.DEFAULT_VERSION);
svc.refresh(info);
- DiskBalancerVolumeChoosingPolicy volumePolicy = mock(DiskBalancerVolumeChoosingPolicy.class);
ContainerChoosingPolicy containerPolicy = mock(ContainerChoosingPolicy.class);
- svc.setVolumeChoosingPolicy(volumePolicy);
- svc.setContainerChoosingPolicy(containerPolicy);
+ svc.setVolumeContainerChoosingPolicy(containerPolicy);
List<StorageVolume> volumes = volumeSet.getVolumesList();
HddsVolume source = (HddsVolume) volumes.get(0);
@@ -335,9 +328,8 @@ public void testConcurrentTasksNotExceedThreadLimit() throws Exception {
when(containerData.getContainerID()).thenAnswer(invocation -> System.nanoTime());
when(containerData.getBytesUsed()).thenReturn(100L);
- when(volumePolicy.chooseVolume(any(), anyDouble(), any(), anyLong())).thenReturn(Pair.of(source, dest));
- when(containerPolicy.chooseContainer(any(), any(), any(), any(), anyDouble(), any(), any()))
- .thenReturn(containerData);
+ when(containerPolicy.chooseVolumesAndContainer(any(), any(), any(), any(), anyDouble()))
+ .thenReturn(new ContainerCandidate(containerData, source, dest));
// Test when no tasks are in progress, it should schedule up to the limit
BackgroundTaskQueue queue = svc.getTasks();
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerTask.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerTask.java
index cd53404a670..0dc92bef87a 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerTask.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerTask.java
@@ -51,7 +51,6 @@
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.fs.MockSpaceUsageCheckFactory;
import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
@@ -564,11 +563,8 @@ public void testDestVolumeCommittedSpaceReleased(ContainerTestVersionInfo versio
GenericTestUtils.LogCapturer serviceLog = GenericTestUtils.LogCapturer.captureLogs(DiskBalancerService.class);
DiskBalancerService.DiskBalancerTask task = getTask();
- long defaultContainerSize = (long) conf.getStorageSize(
- ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
- ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
- // verify committed space is reserved for destination volume
- assertEquals(defaultContainerSize, destVolume.getCommittedBytes() - initialDestCommitted);
+ // verify committed space is reserved for destination volume (uses actual container size)
+ assertEquals(CONTAINER_SIZE, destVolume.getCommittedBytes() - initialDestCommitted);
// delete the container from containerSet to simulate a failure
containerSet.removeContainer(CONTAINER_ID);
@@ -626,7 +622,7 @@ public void testMoveSkippedWhenContainerStateChanged(State invalidState)
throws IOException, InterruptedException, TimeoutException {
LogCapturer serviceLog = LogCapturer.captureLogs(DiskBalancerService.class);
- // Create a CLOSED container which will be selected by DefaultContainerChoosingPolicy
+ // Create a CLOSED container which will be selected by DefaultVolumeContainerChoosingPolicy
Container container = createContainer(CONTAINER_ID, sourceVolume, State.CLOSED);
long initialSourceUsed = sourceVolume.getCurrentUsage().getUsedSpace();
long initialDestUsed = destVolume.getCurrentUsage().getUsedSpace();
diff --git a/hadoop-hdds/docs/content/design/diskbalancer.md b/hadoop-hdds/docs/content/design/diskbalancer.md
index aab8af2ff34..ab1547c4c49 100644
--- a/hadoop-hdds/docs/content/design/diskbalancer.md
+++ b/hadoop-hdds/docs/content/design/diskbalancer.md
@@ -120,16 +120,10 @@ and balanced state will be visible only after the configured delay, when the sou
By default, the DiskBalancer uses specific policies to decide which disks to balance and which containers to move. These
are configurable, but the default implementations provide robust and safe behavior.
-* **`DefaultVolumeChoosingPolicy`**: This is the default policy for selecting the source and destination volumes. It
-identifies the most over-utilized volume as the source and the most under-utilized volume as the destination by comparing
-each volume's utilization against the Datanode's average. The calculation is smart enough to account for data that is
-already in the process of being moved, ensuring it makes accurate decisions based on the future state of the volumes.
-
-* **`DefaultContainerChoosingPolicy`**: This is the default policy for selecting which container to move from a source
-volume. It iterates through the containers on the source disk and picks the first one that is in a **CLOSED** state
-and is not already being moved by another balancing operation. To optimize performance and avoid re-scanning the same
-containers repeatedly, it caches the list of containers for each volume which auto expires after one hour of its last
-used time or if the container iterator for that is invalidated on full utilisation.
+* **`DefaultContainerChoosingPolicy`**: This is the default policy that consolidates both volume selection and container
+selection into a single operation. It identifies the most over-utilized volume as the source and the most under-utilized
+volume with sufficient space as the destination, then iterates through containers on the source to pick the first one
+that is in a **CLOSED** state and is not already being moved. It caches the list of containers for each volume which auto expires after one hour.
## Security Design
DiskBalancer follows the same security model as other services:
diff --git a/hadoop-hdds/docs/content/feature/DiskBalancer.md b/hadoop-hdds/docs/content/feature/DiskBalancer.md
index 317d3f02372..65079064f90 100644
--- a/hadoop-hdds/docs/content/feature/DiskBalancer.md
+++ b/hadoop-hdds/docs/content/feature/DiskBalancer.md
@@ -238,17 +238,16 @@ ozone admin datanode diskbalancer report --in-service-datanodes --json
The DiskBalancer's behavior can be controlled using the following configuration properties in `ozone-site.xml`.
-| Property | Default Value | Description |
-|-------------------------------------------------------------|----------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `hdds.datanode.disk.balancer.enabled` | `false` | If false, the DiskBalancer service on the Datanode is disabled. Configure it to true for diskBalancer to be enabled. |
-| `hdds.datanode.disk.balancer.volume.density.threshold.percent` | `10.0` | A percentage (0-100). A datanode is considered balanced if for each volume, its utilization differs from the average datanode utilization by no more than this threshold. |
-| `hdds.datanode.disk.balancer.max.disk.throughputInMBPerSec` | `10` | The maximum bandwidth (in MB/s) that the balancer can use for moving data, to avoid impacting client I/O. |
-| `hdds.datanode.disk.balancer.parallel.thread` | `5` | The number of worker threads to use for moving containers in parallel. |
-| `hdds.datanode.disk.balancer.service.interval` | `60s` | The time interval at which the Datanode DiskBalancer service checks for imbalance and updates its configuration. |
-| `hdds.datanode.disk.balancer.stop.after.disk.even` | `true` | If true, the DiskBalancer will automatically stop its balancing activity once disks are considered balanced (i.e., all volume densities are within the threshold). |
+| Property | Default Value | Description |
+|-------------------------------------------------------------|----------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `hdds.datanode.disk.balancer.enabled` | `false` | If false, the DiskBalancer service on the Datanode is disabled. Configure it to true for diskBalancer to be enabled. |
+| `hdds.datanode.disk.balancer.volume.density.threshold.percent` | `10.0` | A percentage (0-100). A datanode is considered balanced if for each volume, its utilization differs from the average datanode utilization by no more than this threshold. |
+| `hdds.datanode.disk.balancer.max.disk.throughputInMBPerSec` | `10` | The maximum bandwidth (in MB/s) that the balancer can use for moving data, to avoid impacting client I/O. |
+| `hdds.datanode.disk.balancer.parallel.thread` | `5` | The number of worker threads to use for moving containers in parallel. |
+| `hdds.datanode.disk.balancer.service.interval` | `60s` | The time interval at which the Datanode DiskBalancer service checks for imbalance and updates its configuration. |
+| `hdds.datanode.disk.balancer.stop.after.disk.even` | `true` | If true, the DiskBalancer will automatically stop its balancing activity once disks are considered balanced (i.e., all volume densities are within the threshold). |
| `hdds.datanode.disk.balancer.replica.deletion.delay` | `5m` | The delay after a container is successfully moved from source volume to destination volume before the source container replica is deleted. This lazy deletion provides a grace period before failing the read thread holding the old container replica. Unit: ns, ms, s, m, h, d. |
-| `hdds.datanode.disk.balancer.volume.choosing.policy` | `org.apache.hadoop.ozone.container.diskbalancer.policy.DefaultVolumeChoosingPolicy` | The policy class for selecting source and destination volumes for balancing. |
-| `hdds.datanode.disk.balancer.container.choosing.policy` | `org.apache.hadoop.ozone.container.diskbalancer.policy.DefaultContainerChoosingPolicy` | The policy class for selecting which containers to move from a source volume to destination volume. |
-| `hdds.datanode.disk.balancer.service.timeout` | `300s` | Timeout for the Datanode DiskBalancer service operations. |
-| `hdds.datanode.disk.balancer.should.run.default` | `false` | If the balancer fails to read its persisted configuration, this value determines if the service should run by default. |
+| `hdds.datanode.disk.balancer.container.choosing.policy` | `org.apache.hadoop.ozone.container.diskbalancer.policy.DefaultContainerChoosingPolicy` | The policy for selecting source/destination volumes and which containers to move. |
+| `hdds.datanode.disk.balancer.service.timeout` | `300s` | Timeout for the Datanode DiskBalancer service operations. |
+| `hdds.datanode.disk.balancer.should.run.default` | `false` | If the balancer fails to read its persisted configuration, this value determines if the service should run by default. |
diff --git a/hadoop-hdds/docs/content/feature/DiskBalancer.zh.md b/hadoop-hdds/docs/content/feature/DiskBalancer.zh.md
index e892ee47e7c..7d610730582 100644
--- a/hadoop-hdds/docs/content/feature/DiskBalancer.zh.md
+++ b/hadoop-hdds/docs/content/feature/DiskBalancer.zh.md
@@ -230,17 +230,16 @@ ozone admin datanode diskbalancer report --in-service-datanodes --json
The DiskBalancer's behavior can be controlled using the following configuration properties in `ozone-site.xml`.
-| Property | Default Value | Description |
-|-------------------------------------------------------------|----------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `hdds.datanode.disk.balancer.enabled` | `false` | 如果为 false,则 Datanode 上的 DiskBalancer 服务将被禁用。将其配置为 true 可启用 DiskBalancer。 | | | [...]
-| `hdds.datanode.disk.balancer.volume.density.threshold.percent` | `10.0` | 百分比(0-100)。如果对于每个卷,其利用率与平均数据节点利用率之差不超过此阈值,则认为数据节点处于平衡状态。 |
-| `hdds.datanode.disk.balancer.max.disk.throughputInMBPerSec` | `10` | 平衡器可用于移动数据的最大带宽(以 MB/s 为单位),以避免影响客户端 I/O。 |
-| `hdds.datanode.disk.balancer.parallel.thread` | `5` | 用于并行移动容器的工作线程数。 |
-| `hdds.datanode.disk.balancer.service.interval` | `60s` | Datanode DiskBalancer 服务检查不平衡并更新其配置的时间间隔。 |
-| `hdds.datanode.disk.balancer.stop.after.disk.even` | `true` | 如果为真,则一旦磁盘被视为平衡(即所有卷密度都在阈值内),DiskBalancer 将自动停止其平衡活动。 |
+| Property | Default Value | Description |
+|-------------------------------------------------------------|----------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `hdds.datanode.disk.balancer.enabled` | `false` | 如果为 false,则 Datanode 上的 DiskBalancer 服务将被禁用。将其配置为 true 可启用 DiskBalancer。 | | | [...]
+| `hdds.datanode.disk.balancer.volume.density.threshold.percent` | `10.0` | 百分比(0-100)。如果对于每个卷,其利用率与平均数据节点利用率之差不超过此阈值,则认为数据节点处于平衡状态。 |
+| `hdds.datanode.disk.balancer.max.disk.throughputInMBPerSec` | `10` | 平衡器可用于移动数据的最大带宽(以 MB/s 为单位),以避免影响客户端 I/O。 |
+| `hdds.datanode.disk.balancer.parallel.thread` | `5` | 用于并行移动容器的工作线程数。 |
+| `hdds.datanode.disk.balancer.service.interval` | `60s` | Datanode DiskBalancer 服务检查不平衡并更新其配置的时间间隔。 |
+| `hdds.datanode.disk.balancer.stop.after.disk.even` | `true` | 如果为真,则一旦磁盘被视为平衡(即所有卷密度都在阈值内),DiskBalancer 将自动停止其平衡活动。 |
| `hdds.datanode.disk.balancer.replica.deletion.delay` | `5m` | 容器成功从源卷移动到目标卷后,源容器副本被删除前的延迟时间。这种延迟删除机制旨在避免旧副本的即时删除导致持有旧容器副本的线程数据读取失败。单位:ns、ms、s、m、h、d。|
-| `hdds.datanode.disk.balancer.volume.choosing.policy` | `org.apache.hadoop.ozone.container.diskbalancer.policy.DefaultVolumeChoosingPolicy` | 用于选择平衡的源卷和目标卷的策略类。 |
-| `hdds.datanode.disk.balancer.container.choosing.policy` | `org.apache.hadoop.ozone.container.diskbalancer.policy.DefaultContainerChoosingPolicy` | 用于选择将哪些容器从源卷移动到目标卷的策略类。 |
-| `hdds.datanode.disk.balancer.service.timeout` | `300s` | Datanode DiskBalancer 服务操作超时。 |
-| `hdds.datanode.disk.balancer.should.run.default` | `false` | 如果平衡器无法读取其持久配置,则该值决定服务是否应默认运行。 |
+| `hdds.datanode.disk.balancer.volume.container.choosing.policy` | `org.apache.hadoop.ozone.container.diskbalancer.policy.DefaultContainerChoosingPolicy` | 用于选择源/目标卷以及要移动的容器的策略。 |
+| `hdds.datanode.disk.balancer.service.timeout` | `300s` | Datanode DiskBalancer 服务操作超时。 |
+| `hdds.datanode.disk.balancer.should.run.default` | `false` | 如果平衡器无法读取其持久配置,则该值决定服务是否应默认运行。 |
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestContainerChoosingPolicy.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestContainerChoosingPolicy.java
deleted file mode 100644
index b9d5989fcab..00000000000
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestContainerChoosingPolicy.java
+++ /dev/null
@@ -1,300 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm.node;
-
-import static org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils.newContainerSet;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-
-import java.io.IOException;
-import java.nio.file.Path;
-import java.time.Duration;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.stream.Collectors;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.fs.MockSpaceUsageCheckFactory;
-import org.apache.hadoop.hdds.fs.MockSpaceUsageSource;
-import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory;
-import org.apache.hadoop.hdds.fs.SpaceUsagePersistence;
-import org.apache.hadoop.hdds.fs.SpaceUsageSource;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto;
-import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.ozone.container.ContainerTestHelper;
-import org.apache.hadoop.ozone.container.common.impl.ContainerData;
-import org.apache.hadoop.ozone.container.common.impl.ContainerDataScanOrder;
-import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
-import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
-import org.apache.hadoop.ozone.container.common.interfaces.Container;
-import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
-import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
-import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
-import org.apache.hadoop.ozone.container.diskbalancer.policy.ContainerChoosingPolicy;
-import org.apache.hadoop.ozone.container.diskbalancer.policy.DefaultContainerChoosingPolicy;
-import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
-import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
-import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
-import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.Timeout;
-import org.junit.jupiter.api.io.TempDir;
-
-/**
- * This class tests the ContainerChoosingPolicy.
- */
-public class TestContainerChoosingPolicy {
-
- private static final int NUM_VOLUMES = 20;
- private static final int NUM_CONTAINERS = 100000;
- private static final int NUM_THREADS = 10;
- private static final int NUM_ITERATIONS = 10000;
- private static final int MAX_IN_PROGRESS = 100;
- private static final double THRESHOLD = 10.0;
-
- private static final OzoneConfiguration CONF = new OzoneConfiguration();
-
- @TempDir
- private Path baseDir;
-
- private List<HddsVolume> volumes;
- private ContainerSet containerSet;
- private OzoneContainer ozoneContainer;
- private ContainerChoosingPolicy containerChoosingPolicy;
- private ExecutorService executor;
- private MutableVolumeSet volumeSet;
-
- // Simulate containers currently being balanced (in progress)
- private Set<ContainerID> inProgressContainerIDs = ConcurrentHashMap.newKeySet();
-
- @BeforeEach
- public void setup() throws Exception {
- containerSet = newContainerSet();
- createVolumes();
- createContainers();
- ozoneContainer = mock(OzoneContainer.class);
- ContainerController containerController = new ContainerController(containerSet, null);
- when(ozoneContainer.getController()).thenReturn(containerController);
- when(ozoneContainer.getContainerSet()).thenReturn(containerSet);
- containerChoosingPolicy = new DefaultContainerChoosingPolicy();
- executor = Executors.newFixedThreadPool(NUM_THREADS);
-
- // Create a spied MutableVolumeSet and inject the test volumes
- String datanodeUuid = UUID.randomUUID().toString();
- volumeSet = spy(new MutableVolumeSet(datanodeUuid, CONF, null,
- StorageVolume.VolumeType.DATA_VOLUME, null));
- when(volumeSet.getVolumesList())
- .thenReturn(new ArrayList<>(volumes));
- }
-
- @AfterEach
- public void cleanUp() {
- volumes.forEach(HddsVolume::shutdown);
-
- // Shutdown executor service
- if (executor != null && !executor.isShutdown()) {
- executor.shutdownNow();
- }
-
- // Clear in-progress container IDs
- inProgressContainerIDs.clear();
-
- // Clear ContainerSet
- containerSet = null;
- }
-
- @Test
- @Timeout(300)
- public void testConcurrentContainerChoosingPerformance() throws Exception {
- testPolicyPerformance("ContainerChoosingPolicy", containerChoosingPolicy);
- }
-
- @Test
- public void testContainerDeletionAfterIteratorGeneration() throws Exception {
- HddsVolume volume = volumes.get(0);
- HddsVolume destVolume = volumes.get(1);
-
- List<Container<?>> containerList = ozoneContainer.getContainerSet().getContainerMap().values().stream()
- .filter(x -> volume.getStorageID().equals(x.getContainerData().getVolume().getStorageID()))
- .filter(x -> x.getContainerData().isClosed())
- .sorted(ContainerDataScanOrder.INSTANCE)
- .collect(Collectors.toList());
- inProgressContainerIDs.clear();
-
- ContainerData container = containerChoosingPolicy.chooseContainer(ozoneContainer, volume, destVolume,
- inProgressContainerIDs, THRESHOLD, volumeSet, null);
- assertNotNull(container);
- assertEquals(containerList.get(0).getContainerData().getContainerID(), container.getContainerID());
-
- ozoneContainer.getContainerSet().removeContainer(containerList.get(1).getContainerData().getContainerID());
- inProgressContainerIDs.add(ContainerID.valueOf(container.getContainerID()));
- container = containerChoosingPolicy.chooseContainer(ozoneContainer, volume,
- destVolume, inProgressContainerIDs, THRESHOLD, volumeSet, null);
- assertEquals(containerList.get(1).getContainerData().getContainerID(), container.getContainerID());
- }
-
- /**
- * SuccessCount: Number of successful container choices from the policy.
- * FailureCount: Failures due to any exceptions thrown during container choice.
- */
- private void testPolicyPerformance(String policyName, ContainerChoosingPolicy policy) throws Exception {
- CountDownLatch latch = new CountDownLatch(NUM_THREADS);
- AtomicInteger containerChosenCount = new AtomicInteger(0);
- AtomicInteger containerNotChosenCount = new AtomicInteger(0);
- AtomicInteger failureCount = new AtomicInteger(0);
- AtomicLong totalTimeNanos = new AtomicLong(0);
-
- Random rand = new Random();
-
- for (int i = 0; i < NUM_THREADS; i++) {
- executor.submit(() -> {
- try {
- long threadStart = System.nanoTime();
- int containerChosen = 0;
- int containerNotChosen = 0;
- int failures = 0;
- // Choose a random volume
- HddsVolume srcVolume = volumes.get(rand.nextInt(NUM_VOLUMES));
- HddsVolume destVolume;
-
- do {
- destVolume = volumes.get(rand.nextInt(NUM_VOLUMES));
- } while (srcVolume.equals(destVolume));
-
- for (int j = 0; j < NUM_ITERATIONS; j++) {
- try {
- ContainerData c = policy.chooseContainer(ozoneContainer, srcVolume,
- destVolume, inProgressContainerIDs, THRESHOLD, volumeSet, null);
- if (c == null) {
- containerNotChosen++;
- } else {
- containerChosen++;
- if (inProgressContainerIDs.size() < MAX_IN_PROGRESS) {
- inProgressContainerIDs.add(ContainerID.valueOf(c.getContainerID()));
- }
- }
- } catch (Exception e) {
- failures++;
- }
- }
-
- long threadEnd = System.nanoTime();
- totalTimeNanos.addAndGet(threadEnd - threadStart);
- containerChosenCount.addAndGet(containerChosen);
- containerNotChosenCount.addAndGet(containerNotChosen);
- failureCount.addAndGet(failures);
- } finally {
- latch.countDown();
- }
- });
- }
-
- // Wait max 5 minutes for test completion
- assertTrue(latch.await(5, TimeUnit.MINUTES), "Test timed out");
-
- long totalOperations = (long) NUM_THREADS * NUM_ITERATIONS;
- double avgTimePerOp = (double) totalTimeNanos.get() / totalOperations;
- double opsPerSec = totalOperations / (totalTimeNanos.get() / 1_000_000_000.0);
-
- System.out.println("Performance results for " + policyName);
- System.out.println("Total volumes: " + NUM_VOLUMES);
- System.out.println("Total containers: " + NUM_CONTAINERS);
- System.out.println("Total threads: " + NUM_THREADS);
- System.out.println("Total operations: " + totalOperations);
- System.out.println("Container Chosen operations: " + containerChosenCount.get());
- System.out.println("Container Not Chosen operations: " + containerNotChosenCount.get());
- System.out.println("Failed operations: " + failureCount.get());
- System.out.println("Total time (ms): " + totalTimeNanos.get() / 1_000_000);
- System.out.println("Average time per operation (ns): " + avgTimePerOp);
- System.out.println("Operations per second: " + opsPerSec);
- }
-
- public void createVolumes() throws IOException {
- // Create volumes with mocked space usage
- volumes = new ArrayList<>();
- for (int i = 0; i < NUM_VOLUMES; i++) {
- String volumePath = baseDir.resolve("disk" + i).toString();
- SpaceUsageSource source = MockSpaceUsageSource.fixed(1000000000, 1000000000 - i * 50000);
- SpaceUsageCheckFactory factory = MockSpaceUsageCheckFactory.of(
- source, Duration.ZERO, SpaceUsagePersistence.None.INSTANCE);
- HddsVolume volume = new HddsVolume.Builder(volumePath)
- .conf(CONF)
- .usageCheckFactory(factory)
- .build();
- volumes.add(volume);
- }
- }
-
- public void createContainers() {
- List<ContainerID> closedContainerIDs = new ArrayList<>();
- Random random = new Random();
- long startTime = System.currentTimeMillis();
-
- for (int i = 0; i < NUM_CONTAINERS; i++) {
- boolean isOpen = i < 10; // First 10 containers are open
- int volumeIndex = i % NUM_VOLUMES; // Distribute containers across volumes
- HddsVolume volume = volumes.get(volumeIndex);
-
- KeyValueContainerData containerData = new KeyValueContainerData(
- i, ContainerLayoutVersion.FILE_PER_BLOCK, ContainerTestHelper.CONTAINER_MAX_SIZE,
- UUID.randomUUID().toString(), UUID.randomUUID().toString());
-
- containerData.setState(isOpen ? ContainerDataProto.State.OPEN : ContainerDataProto.State.CLOSED);
- containerData.setVolume(volume);
- // Set some bytes used for containers so they can be chosen for disk balancing
- // Use a small non-zero value to ensure containers are not skipped
- long bytesUsed = isOpen ? 0 : (i % 1000 + 1) * 1024L; // 1KB to 1MB for closed containers
- containerData.getStatistics().setBlockBytesForTesting(bytesUsed);
- KeyValueContainer container = new KeyValueContainer(containerData, CONF);
-
- try {
- containerSet.addContainer(container); // Add container to ContainerSet
- } catch (Exception e) {
- Assertions.fail(e.getMessage());
- }
-
- // Collect IDs of closed containers
- if (!isOpen) {
- closedContainerIDs.add(ContainerID.valueOf((long) i));
- }
- }
-
- // Randomly select NUM_THREADS closed containers to be in-progress
- Collections.shuffle(closedContainerIDs, random);
- inProgressContainerIDs.addAll(closedContainerIDs.subList(0, NUM_THREADS));
- System.out.println("Created " + NUM_CONTAINERS + " containers in " +
- (System.currentTimeMillis() - startTime) + " ms");
- }
-}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancerPolicyPerformance.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancerPolicyPerformance.java
new file mode 100644
index 00000000000..1b621629e7b
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancerPolicyPerformance.java
@@ -0,0 +1,376 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.scm.node;
+
+import static org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils.newContainerSet;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantLock;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.fs.MockSpaceUsageCheckFactory;
+import org.apache.hadoop.hdds.fs.MockSpaceUsageSource;
+import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory;
+import org.apache.hadoop.hdds.fs.SpaceUsagePersistence;
+import org.apache.hadoop.hdds.fs.SpaceUsageSource;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
+import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
+import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
+import org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerVolumeCalculation;
+import org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerVolumeCalculation.VolumeFixedUsage;
+import org.apache.hadoop.ozone.container.diskbalancer.policy.ContainerCandidate;
+import org.apache.hadoop.ozone.container.diskbalancer.policy.ContainerChoosingPolicy;
+import org.apache.hadoop.ozone.container.diskbalancer.policy.DefaultContainerChoosingPolicy;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * This class tests the DiskBalancer ContainerChoosingPolicy.
+ */
+public class TestDiskBalancerPolicyPerformance {
+
+ private static final int NUM_VOLUMES = 20;
+ private static final int NUM_CONTAINERS = 100000;
+ private static final int NUM_THREADS = 10;
+ private static final int NUM_ITERATIONS = 10000;
+ private static final int MAX_IN_PROGRESS = 2500;
+ private static final double THRESHOLD = 10.0;
+
+ private OzoneConfiguration conf;
+
+ @TempDir
+ private Path baseDir;
+
+ private List<HddsVolume> volumes;
+ private ContainerSet containerSet;
+ private OzoneContainer ozoneContainer;
+ private ContainerChoosingPolicy policy;
+ private ExecutorService executor;
+ private MutableVolumeSet volumeSet;
+
+ // Simulate containers currently being balanced (in progress)
+ private Set<ContainerID> inProgressContainerIDs = ConcurrentHashMap.newKeySet();
+ private Map<HddsVolume, Long> deltaMap = new ConcurrentHashMap<>();
+
+ @BeforeEach
+ public void setup() throws Exception {
+ conf = new OzoneConfiguration();
+ conf.set("hdds.datanode.volume.min.free.space", "20GB");
+ conf.setFloat("hdds.datanode.volume.min.free.space.percent", 0.02f);
+
+ volumes = new ArrayList<>();
+ containerSet = newContainerSet();
+ createVolumes();
+ createContainers();
+ ozoneContainer = mock(OzoneContainer.class);
+ ContainerController containerController = new ContainerController(containerSet, null);
+ when(ozoneContainer.getController()).thenReturn(containerController);
+ when(ozoneContainer.getContainerSet()).thenReturn(containerSet);
+ policy = new DefaultContainerChoosingPolicy(new ReentrantLock());
+ executor = Executors.newFixedThreadPool(NUM_THREADS);
+ }
+
+ @AfterEach
+ public void cleanUp() {
+ volumes.forEach(HddsVolume::shutdown);
+
+ // Shutdown executor service
+ if (executor != null && !executor.isShutdown()) {
+ executor.shutdownNow();
+ }
+
+ // Clear in-progress container IDs
+ inProgressContainerIDs.clear();
+
+ // Clear ContainerSet
+ containerSet = null;
+
+ if (volumeSet != null) {
+ volumeSet.shutdown();
+ volumeSet = null;
+ }
+
+
+ }
+
+ // ---------- Volume selection tests (chooseVolumesAndContainer) ----------
+
+ @Test
+ @Timeout(30)
+ public void testVolumeChoosingFailureDueToDiskFull() {
+ OzoneConfiguration testConf = new OzoneConfiguration();
+ testConf.set("hdds.datanode.volume.min.free.space", "990GB");
+ for (StorageVolume volume : volumeSet.getVolumesList()) {
+ volume.setConf(testConf);
+ }
+ assertNull(policy.chooseVolumesAndContainer(ozoneContainer, volumeSet,
+ deltaMap, inProgressContainerIDs, THRESHOLD));
+ assertEquals(NUM_VOLUMES, volumeSet.getVolumesList().size());
+ }
+
+ @Test
+ @Timeout(300)
+ public void testConcurrentContainerChoosingPerformance() throws Exception {
+ testDiskBalancerPolicyPerformance("VolumeAndContainerSelection", policy);
+ }
+
+ private void testDiskBalancerPolicyPerformance(String testName, ContainerChoosingPolicy pol) throws Exception {
+ CountDownLatch latch = new CountDownLatch(NUM_THREADS);
+ AtomicInteger pairChosenCount = new AtomicInteger(0);
+ AtomicInteger pairNotChosenCount = new AtomicInteger(0);
+ AtomicInteger failureCount = new AtomicInteger(0);
+ AtomicLong totalTimeNanos = new AtomicLong(0);
+
+ for (int i = 0; i < NUM_THREADS; i++) {
+ executor.submit(() -> {
+ try {
+ int containerCandidateChosen = 0;
+ int containerCandidateNotChosen = 0;
+ int failures = 0;
+
+ for (int j = 0; j < NUM_ITERATIONS; j++) {
+ long threadStart = System.nanoTime();
+ try {
+ ContainerCandidate candidate = pol.chooseVolumesAndContainer(ozoneContainer,
+ volumeSet, deltaMap, inProgressContainerIDs, THRESHOLD);
+
+ if (candidate == null) {
+ containerCandidateNotChosen++;
+ } else {
+ containerCandidateChosen++;
+ ContainerID chosenId = ContainerID.valueOf(candidate.getContainerData().getContainerID());
+ if (inProgressContainerIDs.size() < MAX_IN_PROGRESS) {
+ inProgressContainerIDs.add(chosenId);
+ }
+ deltaMap.compute(candidate.getSourceVolume(), (k, v) -> (v == null ? 0L : v)
+ - candidate.getContainerData().getBytesUsed());
+ }
+ } catch (Exception e) {
+ failures++;
+ } finally {
+ totalTimeNanos.addAndGet(System.nanoTime() - threadStart);
+ }
+ }
+
+ pairChosenCount.addAndGet(containerCandidateChosen);
+ pairNotChosenCount.addAndGet(containerCandidateNotChosen);
+ failureCount.addAndGet(failures);
+ } finally {
+ latch.countDown();
+ }
+ });
+ }
+
+ assertTrue(latch.await(5, TimeUnit.MINUTES), "Test timed out");
+
+ long totalOperations = (long) NUM_THREADS * NUM_ITERATIONS;
+ int selections = pairChosenCount.get();
+ int noSelection = pairNotChosenCount.get();
+ System.out.println("Performance results for " + testName);
+ System.out.println("Total volumes: " + NUM_VOLUMES);
+ System.out.println("Total containers: " + NUM_CONTAINERS);
+ System.out.println("Total threads: " + NUM_THREADS);
+ System.out.println("Total operations: " + totalOperations);
+ System.out.println("Container Candidate Chosen (src+dest+container): " + selections);
+ System.out.println("No Container Candidate Chosen: " + noSelection);
+ System.out.println("Failed: " + failureCount.get());
+ System.out.println("Total time (ms): " + totalTimeNanos.get() / 1_000_000);
+ System.out.println("Avg time per operation (ns): " + (double) totalTimeNanos.get() / totalOperations);
+ System.out.println("Operations per second: " + totalOperations / (totalTimeNanos.get() / 1_000_000_000.0));
+ }
+
+ /**
+ * Tests that chooseVolumesAndContainer skips in-progress containers and handles
+ * containers removed from containerSet during iteration.
+ */
+ @Test
+ public void testContainerDeletionAfterIteratorGeneration() throws Exception {
+ inProgressContainerIDs.clear();
+ deltaMap.clear();
+
+ ContainerCandidate first = policy.chooseVolumesAndContainer(ozoneContainer,
+ volumeSet, deltaMap, inProgressContainerIDs, THRESHOLD);
+ assertNotNull(first, "Expected to choose a container on first call");
+ long firstContainerId = first.getContainerData().getContainerID();
+
+ inProgressContainerIDs.add(ContainerID.valueOf(firstContainerId));
+
+ List<Long> sourceContainerIds = getSourceContainerIdsInPolicyOrder();
+ long secondId = sourceContainerIds.stream()
+ .filter(id -> id != firstContainerId)
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("Expected at least 2 containers on source"));
+ containerSet.removeContainerOnlyFromMemory(secondId);
+
+ ContainerCandidate second = policy.chooseVolumesAndContainer(ozoneContainer,
+ volumeSet, deltaMap, inProgressContainerIDs, THRESHOLD);
+ assertNotNull(second, "Expected to choose a container on second call");
+ assertTrue(second.getContainerData().getContainerID() != firstContainerId
+ && second.getContainerData().getContainerID() != secondId,
+ "Should skip in-progress and deleted, return third container");
+ }
+
+ private List<Long> getSourceContainerIdsInPolicyOrder() {
+ List<Long> ids = new ArrayList<>();
+ List<VolumeFixedUsage> usages = DiskBalancerVolumeCalculation.getVolumeUsages(volumeSet, deltaMap);
+ usages.stream()
+ .max(Comparator.comparingDouble(VolumeFixedUsage::getUtilization)
+ .thenComparing(v -> v.getVolume().getStorageID()))
+ .map(VolumeFixedUsage::getVolume)
+ .ifPresent(src -> ozoneContainer.getController().getContainers(src)
+ .forEachRemaining(c -> {
+ if (c.getContainerData().isClosed() && c.getContainerData().getBytesUsed() > 0) {
+ ids.add(c.getContainerData().getContainerID());
+ }
+ }));
+ return ids;
+ }
+
+ private void createVolumes() throws IOException {
+ String initialDnDir = baseDir.resolve("initialDnDir").toFile().getAbsolutePath();
+ Files.createDirectories(baseDir.resolve("initialDnDir"));
+ conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, initialDnDir);
+
+ StateContext mockContext = mock(StateContext.class);
+ volumeSet = new MutableVolumeSet(UUID.randomUUID().toString(), conf,
+ mockContext, StorageVolume.VolumeType.DATA_VOLUME, null);
+
+ Map<String, StorageVolume> newVolumeMap = new ConcurrentHashMap<>();
+ Random random = new Random();
+ long startTime = System.currentTimeMillis();
+
+ for (int i = 0; i < NUM_VOLUMES; i++) {
+ String volumePath = baseDir.resolve("disk" + i).toString();
+ long capacity = 1L * 1024 * 1024 * 1024 * 1024; // 1TB
+
+ // Vary utilization from 5% to 55% so we have imbalance to balance
+ double utilFraction = 0.05 + (i / (double) NUM_VOLUMES) * 0.5 + random.nextDouble() * 0.05;
+ long usedBytes = (long) (capacity * Math.min(utilFraction, 0.55));
+ long available = capacity - usedBytes;
+
+ SpaceUsageSource source = MockSpaceUsageSource.fixed(capacity, available);
+ SpaceUsageCheckFactory factory = MockSpaceUsageCheckFactory.of(
+ source, Duration.ZERO, SpaceUsagePersistence.None.INSTANCE);
+ HddsVolume volume = new HddsVolume.Builder(volumePath)
+ .conf(conf)
+ .usageCheckFactory(factory)
+ .build();
+ volume.setConf(conf);
+ volumes.add(volume);
+ newVolumeMap.put(volume.getStorageDir().getPath(), volume);
+ }
+
+ volumeSet.setVolumeMapForTesting(newVolumeMap);
+ System.out.println("Created " + NUM_VOLUMES + " volumes in " +
+ (System.currentTimeMillis() - startTime) + " ms");
+ }
+
+ private void createContainers() {
+ List<ContainerID> closedContainerIDs = new ArrayList<>();
+ Random random = new Random();
+ long startTime = System.currentTimeMillis();
+
+ for (int i = 0; i < NUM_CONTAINERS; i++) {
+ // ~50 OPEN containers (policy skips) - keep few so most are eligible
+ boolean isOpen = i < 50;
+ // High-util volumes (index 15-19) get more containers - more source candidates
+ int volumeIndex = (i % 2 == 0) ? (i % NUM_VOLUMES) : (15 + (i % 5));
+ volumeIndex = Math.min(volumeIndex, NUM_VOLUMES - 1);
+ HddsVolume volume = volumes.get(volumeIndex);
+
+ KeyValueContainerData containerData = new KeyValueContainerData(
+ i, ContainerLayoutVersion.FILE_PER_BLOCK, ContainerTestHelper.CONTAINER_MAX_SIZE,
+ UUID.randomUUID().toString(), UUID.randomUUID().toString());
+
+ containerData.setState(isOpen ? ContainerDataProto.State.OPEN : ContainerDataProto.State.CLOSED);
+ containerData.setVolume(volume);
+
+ // Varied sizes: small (1-10MB), medium (20-80MB), large (100-400MB)
+ long bytesUsed;
+ if (isOpen) {
+ bytesUsed = 0;
+ } else {
+ int sizeIdx = i % 100;
+ if (sizeIdx < 2) {
+ bytesUsed = 0; // Policy skips size 0 (~2% of closed)
+ } else if (sizeIdx < 60) {
+ bytesUsed = (1 + (i % 10)) * 1024 * 1024L; // 1-10 MB
+ } else if (sizeIdx < 90) {
+ bytesUsed = (20 + (i % 60)) * 1024 * 1024L; // 20-80 MB
+ } else {
+ bytesUsed = (100 + (i % 300)) * 1024 * 1024L; // 100-400 MB
+ }
+ }
+ containerData.getStatistics().setBlockBytesForTesting(bytesUsed);
+ KeyValueContainer container = new KeyValueContainer(containerData, conf);
+
+ try {
+ containerSet.addContainer(container); // Add container to ContainerSet
+ } catch (Exception e) {
+ Assertions.fail(e.getMessage());
+ }
+
+ if (!isOpen) {
+ closedContainerIDs.add(ContainerID.valueOf((long) i));
+ }
+ }
+
+ Collections.shuffle(closedContainerIDs, random);
+ inProgressContainerIDs.addAll(closedContainerIDs.subList(0, Math.min(NUM_THREADS, closedContainerIDs.size())));
+ System.out.println("Created " + NUM_CONTAINERS + " containers in " +
+ (System.currentTimeMillis() - startTime) + " ms");
+ }
+}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestVolumeChoosingPolicy.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestVolumeChoosingPolicy.java
deleted file mode 100644
index cdbe046635c..00000000000
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestVolumeChoosingPolicy.java
+++ /dev/null
@@ -1,285 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm.node;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.mockito.Mockito.mock;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.time.Duration;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReentrantLock;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.fs.MockSpaceUsageCheckFactory;
-import org.apache.hadoop.hdds.fs.MockSpaceUsageSource;
-import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory;
-import org.apache.hadoop.hdds.fs.SpaceUsagePersistence;
-import org.apache.hadoop.hdds.fs.SpaceUsageSource;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
-import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
-import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
-import org.apache.hadoop.ozone.container.diskbalancer.policy.DefaultVolumeChoosingPolicy;
-import org.apache.hadoop.ozone.container.diskbalancer.policy.DiskBalancerVolumeChoosingPolicy;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.Timeout;
-import org.junit.jupiter.api.io.TempDir;
-
-/**
- * This class tests the VolumeChoosingPolicy.
- */
-public class TestVolumeChoosingPolicy {
-
- private static final int NUM_VOLUMES = 20;
- private static final int NUM_THREADS = 10;
- private static final int NUM_ITERATIONS = 10000;
- private static final double THRESHOLD = 10; // 10% threshold
- private static final long DEFAULT_CONTAINER_SIZE = 100L * 1024 * 1024; // 100MB
-
- private OzoneConfiguration conf;
-
- @TempDir
- private Path baseDir;
-
- private MutableVolumeSet volumeSet;
- private List<HddsVolume> hddsVolumes;
- private DiskBalancerVolumeChoosingPolicy volumeChoosingPolicy;
- private ExecutorService executor;
-
- // delta sizes for source volumes
- private Map<HddsVolume, Long> deltaSizes = new ConcurrentHashMap<>();
-
- @BeforeEach
- public void setup() throws Exception {
- // Create a fresh configuration for each test to avoid interference between tests
- conf = new OzoneConfiguration();
- hddsVolumes = new ArrayList<>();
- createVolumes();
- volumeChoosingPolicy = new DefaultVolumeChoosingPolicy(new ReentrantLock());
- executor = Executors.newFixedThreadPool(NUM_THREADS);
- }
-
- @AfterEach
- public void cleanUp() {
- hddsVolumes.forEach(HddsVolume::shutdown);
-
- // Shutdown executor service
- if (executor != null && !executor.isShutdown()) {
- executor.shutdownNow();
- }
-
- // Clear in-progress delta sizes
- if (deltaSizes != null) {
- deltaSizes.clear();
- }
-
- // Clear VolumeSet
- if (volumeSet != null) {
- volumeSet.shutdown();
- volumeSet = null;
- }
- }
-
- @Test
- @Timeout(30)
- public void testVolumeChoosingFailureDueToDiskFull() {
- // Create a separate configuration for this test to avoid affecting other tests
- OzoneConfiguration testConf = new OzoneConfiguration();
- // update volume configure, set a huge min free space
- testConf.set("hdds.datanode.volume.min.free.space", "990GB");
- for (StorageVolume volume: volumeSet.getVolumesList()) {
- volume.setConf(testConf);
- }
- assertNull(volumeChoosingPolicy.chooseVolume(volumeSet, THRESHOLD, deltaSizes, DEFAULT_CONTAINER_SIZE));
- assertEquals(NUM_VOLUMES, volumeSet.getVolumesList().size());
- }
-
- @Test
- @Timeout(300)
- public void testConcurrentVolumeChoosingPerformance() throws Exception {
- testPolicyPerformance("DefaultVolumeChoosingPolicy", volumeChoosingPolicy);
- }
-
- /**
- * pairChosenCount: Number of successful volume pair choices from the policy.
- * FailureCount: Failures due to any exceptions thrown during volume choice or null return.
- */
- private void testPolicyPerformance(String policyName, DiskBalancerVolumeChoosingPolicy policy) throws Exception {
- CountDownLatch latch = new CountDownLatch(NUM_THREADS);
- AtomicInteger pairChosenCount = new AtomicInteger(0);
- AtomicInteger pairNotChosenCount = new AtomicInteger(0);
- AtomicInteger failureCount = new AtomicInteger(0);
- AtomicLong totalTimeNanos = new AtomicLong(0);
-
- Random rand = new Random();
-
- for (int i = 0; i < NUM_THREADS; i++) {
- executor.submit(() -> {
- try {
- int volumeChosen = 0;
- int volumeNotChosen = 0;
- int failures = 0;
-
- for (int j = 0; j < NUM_ITERATIONS; j++) {
- // Simulate background activity with a 5% probability in each
- // iteration. This mimics other threads changing volume state
- // (e.g., reserving space on a destination) while the policy is
- // being evaluated, testing its behavior in a dynamic environment.
- if (rand.nextDouble() < 0.05 && hddsVolumes.size() >= 2) {
- HddsVolume sourceVolume = hddsVolumes.get(rand.nextInt(hddsVolumes.size()));
-
- // Randomly pick a destination volume, ensuring it's different from the source
- HddsVolume destVolume;
- do {
- destVolume = hddsVolumes.get(rand.nextInt(hddsVolumes.size()));
- } while (sourceVolume.equals(destVolume));
-
- long containerSize = rand.nextInt(100 * 1024 * 1024) + 1;
-
- // Simulate deltaSizes update for the source volume(negative value)
- deltaSizes.compute(sourceVolume, (k, v) -> (v == null ? 0L : v) - containerSize);
-
- // Simulate committedBytes update for the destination volume (space reserved)
- destVolume.incCommittedBytes(containerSize);
- }
-
- long threadStart = System.nanoTime();
- try {
- // Use a reasonable container size for the policy check
- // The policy checks if containerSize < computeUsableSpace()
- Pair<HddsVolume, HddsVolume> pair = policy.chooseVolume(
- volumeSet, THRESHOLD, deltaSizes, DEFAULT_CONTAINER_SIZE);
-
- if (pair == null) {
- volumeNotChosen++;
- } else {
- volumeChosen++;
- HddsVolume sourceVolume = pair.getLeft();
-
- // The policy automatically calls destVolume.incCommittedBytes(containerSize)
- // when a pair is chosen, so we need to simulate the corresponding
- // deltaMap update for the source volume (negative value = space to be freed)
- // This aligns with how DiskBalancerService updates deltaSizes after choosing
- deltaSizes.compute(sourceVolume, (k, v) -> (v == null ? 0L : v) - DEFAULT_CONTAINER_SIZE);
-
- // Note: In a real DiskBalancerService, after a successful container move,
- // the committed bytes on the destination would be adjusted and the delta
- // on the source would be cleared. For this performance test, we simulate
- // the ongoing state changes that occur during disk balancing operations.
- }
- } catch (Exception e) {
- failures++;
- } finally {
- totalTimeNanos.addAndGet(System.nanoTime() - threadStart);
- }
- }
-
- pairChosenCount.addAndGet(volumeChosen);
- pairNotChosenCount.addAndGet(volumeNotChosen);
- failureCount.addAndGet(failures);
- } finally {
- latch.countDown();
- }
- });
- }
-
- // Wait max 5 minutes for test completion
- assertTrue(latch.await(5, TimeUnit.MINUTES), "Test timed out");
-
- long totalOperations = (long) NUM_THREADS * NUM_ITERATIONS;
- double avgTimePerOp = (double) totalTimeNanos.get() / totalOperations;
- double opsPerSec = totalOperations / (totalTimeNanos.get() / 1_000_000_000.0);
-
- System.out.println("Performance results for " + policyName);
- System.out.println("Total volumes: " + NUM_VOLUMES);
- System.out.println("Total threads: " + NUM_THREADS);
- System.out.println("Threshold(%): " + THRESHOLD);
- System.out.println("Total operations: " + totalOperations);
- System.out.println("Volume Pair Chosen operations: " + pairChosenCount.get());
- System.out.println("Volume Pair Not Chosen operations: " + pairNotChosenCount.get());
- System.out.println("Failed operations: " + failureCount.get());
- System.out.println("Total time (ms): " + totalTimeNanos.get() / 1_000_000);
- System.out.println("Average time per operation (ns): " + avgTimePerOp);
- System.out.println("Operations per second: " + opsPerSec);
- }
-
- private void createVolumes() throws IOException {
- // set a dummy path for initialisation, as we will override its internal volumeMap.
- String initialDnDir = baseDir.resolve("initialDnDir").toFile().getAbsolutePath();
- Files.createDirectories(baseDir.resolve("initialDnDir"));
- conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, initialDnDir);
- conf.set("hdds.datanode.volume.min.free.space", "20GB");
- conf.setFloat("hdds.datanode.volume.min.free.space.percent", 0.02f);
-
- StateContext mockContext = mock(StateContext.class);
- volumeSet = new MutableVolumeSet(UUID.randomUUID().toString(), conf,
- mockContext, StorageVolume.VolumeType.DATA_VOLUME, null);
-
- // This map will replace the one inside 'volumeSet'
- Map<String, StorageVolume> newVolumeMap = new ConcurrentHashMap<>();
- Random random = new Random();
- long startTime = System.currentTimeMillis();
-
- for (int i = 0; i < NUM_VOLUMES; i++) {
- String volumePath = baseDir.resolve("disk" + i).toString();
- //capacity of each volume is 1TB
- long capacity = 1L * 1024 * 1024 * 1024 * 1024;
-
- // Distribute available space to create some variation in usage
- // Between 5% and 55%
- long usedBytes = (long) (capacity * (0.05 + random.nextDouble() * 0.5));
- long available = capacity - usedBytes;
-
- SpaceUsageSource source = MockSpaceUsageSource.fixed(capacity, available);
- SpaceUsageCheckFactory factory = MockSpaceUsageCheckFactory.of(
- source, Duration.ZERO, SpaceUsagePersistence.None.INSTANCE);
- HddsVolume volume = new HddsVolume.Builder(volumePath)
- .conf(conf)
- .usageCheckFactory(factory)
- .build();
- // Ensure configuration is applied to the volume
- volume.setConf(conf);
- hddsVolumes.add(volume);
- newVolumeMap.put(volume.getStorageDir().getPath(), volume);
- }
-
- // Initialize the volumeSet with the new volume map
- volumeSet.setVolumeMapForTesting(newVolumeMap);
- System.out.println("Created " + NUM_VOLUMES + " volumes in " +
- (System.currentTimeMillis() - startTime) + " ms");
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org