You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2021/10/01 18:21:59 UTC
[ozone] branch master updated: HDDS-5797. Support setting Datanode
Reserved Space in MiniOzoneCluster. (#2695)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new c886a2f HDDS-5797. Support setting Datanode Reserved Space in MiniOzoneCluster. (#2695)
c886a2f is described below
commit c886a2fb1deb08ca8c730626fb1909cccb4527ad
Author: Siddhant Sangwan <si...@gmail.com>
AuthorDate: Fri Oct 1 23:51:44 2021 +0530
HDDS-5797. Support setting Datanode Reserved Space in MiniOzoneCluster. (#2695)
---
.../ozone/container/common/volume/VolumeInfo.java | 5 +++++
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 19 +++++++++++++++++++
.../org/apache/hadoop/ozone/MiniOzoneClusterImpl.java | 7 +++++++
.../org/apache/hadoop/ozone/TestMiniOzoneCluster.java | 16 +++++++++++++---
4 files changed, 44 insertions(+), 3 deletions(-)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
index 7e541a6..3679c6a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
@@ -233,4 +233,9 @@ public final class VolumeInfo {
public VolumeUsage getUsageForTesting() {
return usage;
}
+
+ @VisibleForTesting
+ public long getReservedInBytes() {
+ return reservedInBytes;
+ }
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index 9057920..7e11b3c 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -315,6 +315,7 @@ public interface MiniOzoneCluster {
protected Optional<String> omId = Optional.empty();
protected Boolean randomContainerPort = true;
+ protected Optional<String> datanodeReservedSpace = Optional.empty();
protected Optional<Integer> chunkSize = Optional.empty();
protected OptionalInt streamBufferSize = OptionalInt.empty();
protected Optional<Long> streamBufferFlushSize = Optional.empty();
@@ -499,6 +500,24 @@ public interface MiniOzoneCluster {
}
/**
+ * Sets the reserved space
+ * {@link org.apache.hadoop.hdds.scm.ScmConfigKeys}
+ * HDDS_DATANODE_DIR_DU_RESERVED
+ * for each volume in each datanode.
+ * @param reservedSpace String that contains the numeric size value and
+ * ends with a
+ * {@link org.apache.hadoop.hdds.conf.StorageUnit}
+ * suffix. For example, "50GB".
+ * @see org.apache.hadoop.ozone.container.common.volume.VolumeInfo
+ *
+ * @return {@link MiniOzoneCluster} Builder
+ */
+ public Builder setDatanodeReservedSpace(String reservedSpace) {
+ datanodeReservedSpace = Optional.of(reservedSpace);
+ return this;
+ }
+
+ /**
* Sets the chunk size.
*
* @return MiniOzoneCluster.Builder
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 4b2765b..d9b5b3c 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -814,11 +814,15 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
String datanodeBaseDir = path + "/datanode-" + Integer.toString(i);
Path metaDir = Paths.get(datanodeBaseDir, "meta");
List<String> dataDirs = new ArrayList<>();
+ List<String> reservedSpaceList = new ArrayList<>();
for (int j = 0; j < numDataVolumes; j++) {
Path dir = Paths.get(datanodeBaseDir, "data-" + j, "containers");
Files.createDirectories(dir);
dataDirs.add(dir.toString());
+ datanodeReservedSpace.ifPresent(
+ s -> reservedSpaceList.add(dir + ":" + s));
}
+ String reservedSpaceString = String.join(",", reservedSpaceList);
String listOfDirs = String.join(",", dataDirs);
Path ratisDir = Paths.get(datanodeBaseDir, "data", "ratis");
Path workDir = Paths.get(datanodeBaseDir, "data", "replication",
@@ -828,6 +832,9 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
Files.createDirectories(workDir);
dnConf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.toString());
dnConf.set(DFSConfigKeysLegacy.DFS_DATANODE_DATA_DIR_KEY, listOfDirs);
+ dnConf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, listOfDirs);
+ dnConf.set(ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED,
+ reservedSpaceString);
dnConf.set(OzoneConfigKeys.DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR,
ratisDir.toString());
dnConf.set(OzoneConfigKeys.OZONE_CONTAINER_COPY_WORKDIR,
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
index 8c51f9d..e7436e0 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hdds.DFSConfigKeysLegacy;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.conf.StorageSize;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
@@ -44,6 +45,7 @@ import org.apache.hadoop.ozone.container.common.SCMTestUtils;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine;
+import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
import org.apache.hadoop.ozone.container.ozoneimpl.TestOzoneContainer;
import org.apache.hadoop.test.PathUtils;
import org.apache.hadoop.test.TestGenericTestUtils;
@@ -351,16 +353,24 @@ public class TestMiniOzoneCluster {
*/
@Test (timeout = 60000)
public void testMultipleDataDirs() throws Exception {
- // Start a cluster with 3 DN
+ // Start a cluster with 3 DN and configure reserved space in each DN
+ String reservedSpace = "1B";
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(1)
.setNumDataVolumes(3)
+ .setDatanodeReservedSpace(reservedSpace)
.build();
cluster.waitForClusterToBeReady();
- Assert.assertEquals(3, cluster.getHddsDatanodes().get(0)
+ List<StorageVolume> volumeList = cluster.getHddsDatanodes().get(0)
.getDatanodeStateMachine().getContainer().getVolumeSet()
- .getVolumesList().size());
+ .getVolumesList();
+
+ Assert.assertEquals(3, volumeList.size());
+
+ volumeList.forEach(storageVolume -> Assert.assertEquals(
+ (long) StorageSize.parse(reservedSpace).getValue(),
+ storageVolume.getVolumeInfo().getReservedInBytes()));
}
private static void assertDetailsEquals(DatanodeDetails expected,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org