[ozone] 02/02: HDDS-7407. EC: Block allocation should not be stripped across the EC group (#3882)

[ck...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

ckj pushed a commit to branch ozone-1.3
in repository https://gitbox.apache.org/repos/asf/ozone.git

commit 355377f9525fb5b6af136c1d1803cca65a66d54b
Author: Kaijie Chen <ck...@apache.org>
AuthorDate: Thu Oct 27 10:14:14 2022 +0800

    HDDS-7407. EC: Block allocation should not be stripped across the EC group (#3882)
---
 .../hadoop/hdds/scm/pipeline/WritableECContainerProvider.java | 11 ++++-------
 .../hdds/scm/pipeline/TestWritableECContainerProvider.java    |  5 ++---
 2 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java
index 76e31d1524..7833f150f4 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java
@@ -93,16 +93,13 @@ public class WritableECContainerProvider
   public ContainerInfo getContainer(final long size,
       ECReplicationConfig repConfig, String owner, ExcludeList excludeList)
       throws IOException, TimeoutException {
-    // Bound this at a minimum of 1 byte in case a request is made for a very
-    // small size, which when divided by EC DataNum is zero.
-    long requiredSpace = Math.max(1, size / repConfig.getData());
     synchronized (this) {
       int openPipelineCount = pipelineManager.getPipelineCount(repConfig,
           Pipeline.PipelineState.OPEN);
       if (openPipelineCount < providerConfig.getMinimumPipelines()) {
         try {
           return allocateContainer(
-              repConfig, requiredSpace, owner, excludeList);
+              repConfig, size, owner, excludeList);
         } catch (IOException e) {
           LOG.warn("Unable to allocate a container for {} with {} existing "
               + "containers", repConfig, openPipelineCount, e);
@@ -115,7 +112,7 @@ public class WritableECContainerProvider
 
     PipelineRequestInformation pri =
         PipelineRequestInformation.Builder.getBuilder()
-            .setSize(requiredSpace)
+            .setSize(size)
             .build();
     while (existingPipelines.size() > 0) {
       Pipeline pipeline =
@@ -129,7 +126,7 @@ public class WritableECContainerProvider
         try {
           ContainerInfo containerInfo = getContainerFromPipeline(pipeline);
           if (containerInfo == null
-              || !containerHasSpace(containerInfo, requiredSpace)) {
+              || !containerHasSpace(containerInfo, size)) {
             // This is O(n), which isn't great if there are a lot of pipelines
             // and we keep finding pipelines without enough space.
             existingPipelines.remove(pipeline);
@@ -153,7 +150,7 @@ public class WritableECContainerProvider
     // allocate a new one and usePipelineManagerV2Impl.java it.
     try {
       synchronized (this) {
-        return allocateContainer(repConfig, requiredSpace, owner, excludeList);
+        return allocateContainer(repConfig, size, owner, excludeList);
       }
     } catch (IOException e) {
       LOG.error("Unable to allocate a container for {} after trying all "
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestWritableECContainerProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestWritableECContainerProvider.java
index 09ae77581f..3296188707 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestWritableECContainerProvider.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestWritableECContainerProvider.java
@@ -298,12 +298,11 @@ public class TestWritableECContainerProvider {
     // Update all the containers to make them nearly full, but with enough space
     // for an EC block to be striped across them.
     for (ContainerInfo c : allocatedContainers) {
-      c.setUsedBytes(getMaxContainerSize() - 30 * 1024 * 1024);
+      c.setUsedBytes(getMaxContainerSize() - 90 * 1024 * 1024);
     }
 
     // Get a new container of size 50 and ensure it is one of the original set.
-    // We ask for a space of 50, but as it is stripped across the EC group it
-    // will actually need 50 / dataNum space
+    // We ask for a space of 50 MB, and will actually need 50 MB space.
     ContainerInfo newContainer =
         provider.getContainer(50 * 1024 * 1024, repConfig, OWNER,
             new ExcludeList());


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org