[hadoop] branch trunk updated: HDDS-1559. Include committedBytes to determine Out of Space in VolumeChoosingPolicy. Contributed by Supratim Deka (#841)

[ar...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

arp pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
     new 346c2b7  HDDS-1559. Include committedBytes to determine Out of Space in VolumeChoosingPolicy. Contributed by Supratim Deka (#841)
346c2b7 is described below

commit 346c2b798080cc1f22d6ba85e584141e7dee2c08
Author: supratimdeka <46...@users.noreply.github.com>
AuthorDate: Wed May 29 09:18:14 2019 +0530

    HDDS-1559. Include committedBytes to determine Out of Space in VolumeChoosingPolicy. Contributed by Supratim Deka (#841)
---
 .../volume/RoundRobinVolumeChoosingPolicy.java     |  4 ++-
 .../container/ozoneimpl/TestOzoneContainer.java    | 35 ++++++++++++++++++++++
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/RoundRobinVolumeChoosingPolicy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/RoundRobinVolumeChoosingPolicy.java
index 75c92ec..f503149 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/RoundRobinVolumeChoosingPolicy.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/RoundRobinVolumeChoosingPolicy.java
@@ -58,7 +58,9 @@ public class RoundRobinVolumeChoosingPolicy implements VolumeChoosingPolicy {
 
     while (true) {
       final HddsVolume volume = volumes.get(currentVolumeIndex);
-      long availableVolumeSize = volume.getAvailable();
+      // adjust for remaining capacity in Open containers
+      long availableVolumeSize = volume.getAvailable()
+          - volume.getCommittedBytes();
 
       currentVolumeIndex = (currentVolumeIndex + 1) % volumes.size();
 
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
index e678282..f5ebb49 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
 import org.apache.hadoop.ozone.container.common.helpers.BlockData;
 import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
 import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
@@ -52,6 +53,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.ArrayList;
 
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.DISK_OUT_OF_SPACE;
 import static org.junit.Assert.assertEquals;
 
 /**
@@ -135,6 +137,39 @@ public class TestOzoneContainer {
     verifyCommittedSpace(ozoneContainer);
   }
 
+  @Test
+  public void testContainerCreateDiskFull() throws Exception {
+    volumeSet = new VolumeSet(datanodeDetails.getUuidString(), conf);
+    volumeChoosingPolicy = new RoundRobinVolumeChoosingPolicy();
+    long containerSize = (long) StorageUnit.MB.toBytes(100);
+    boolean diskSpaceException = false;
+
+    // Format the volumes
+    for (HddsVolume volume : volumeSet.getVolumesList()) {
+      volume.format(UUID.randomUUID().toString());
+
+      // eat up all available space except size of 1 container
+      volume.incCommittedBytes(volume.getAvailable() - containerSize);
+      // eat up 10 bytes more, now available space is less than 1 container
+      volume.incCommittedBytes(10);
+    }
+    keyValueContainerData = new KeyValueContainerData(99, containerSize,
+        UUID.randomUUID().toString(), datanodeDetails.getUuidString());
+    keyValueContainer = new KeyValueContainer(keyValueContainerData, conf);
+
+    // we expect an out of space Exception
+    try {
+      keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId);
+    } catch (StorageContainerException e) {
+      if (e.getResult() == DISK_OUT_OF_SPACE) {
+        diskSpaceException = true;
+      }
+    }
+
+    // Test failed if there was no exception
+    assertEquals(true, diskSpaceException);
+  }
+
   //verify committed space on each volume
   private void verifyCommittedSpace(OzoneContainer oc) {
     for (HddsVolume dnVol : oc.getVolumeSet().getVolumesList()) {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org