You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by um...@apache.org on 2021/07/26 15:46:45 UTC
[ozone] branch HDDS-3816-ec updated: HDDS-5174. EC: Allow EC blocks
to be requested from OM (#2414)
This is an automated email from the ASF dual-hosted git repository.
umamahesh pushed a commit to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-3816-ec by this push:
new cd781b9 HDDS-5174. EC: Allow EC blocks to be requested from OM (#2414)
cd781b9 is described below
commit cd781b97859f531ef1498d35e508a19e9be61dec
Author: Stephen O'Donnell <st...@gmail.com>
AuthorDate: Mon Jul 26 16:45:05 2021 +0100
HDDS-5174. EC: Allow EC blocks to be requested from OM (#2414)
---
.../ScmBlockLocationProtocolClientSideTranslatorPB.java | 4 ++++
.../hadoop/ozone/om/request/file/OMFileCreateRequest.java | 5 ++++-
.../ozone/om/request/key/OMAllocateBlockRequest.java | 8 +++++---
.../hadoop/ozone/om/request/key/OMKeyCreateRequest.java | 5 ++++-
.../apache/hadoop/ozone/om/request/key/OMKeyRequest.java | 14 +++++---------
.../hadoop/ozone/om/request/key/TestOMKeyRequest.java | 4 ++--
6 files changed, 24 insertions(+), 16 deletions(-)
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java
index 412b2ec..a24b458 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java
@@ -24,6 +24,7 @@ import java.util.stream.Collectors;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.client.ContainerBlockID;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
@@ -170,6 +171,9 @@ public final class ScmBlockLocationProtocolClientSideTranslatorPB
requestBuilder.setFactor(
((RatisReplicationConfig) replicationConfig).getReplicationFactor());
break;
+ case EC:
+ requestBuilder.setEcReplicationConfig(
+ ((ECReplicationConfig)replicationConfig).toProto());
default:
throw new IllegalArgumentException(
"Unsupported replication type " + replicationConfig
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
index ec7eb87..d0bf379 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
@@ -28,6 +28,7 @@ import java.util.stream.Collectors;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.OzoneConsts;
@@ -124,13 +125,15 @@ public class OMFileCreateRequest extends OMKeyRequest {
type = useRatis ? HddsProtos.ReplicationType.RATIS :
HddsProtos.ReplicationType.STAND_ALONE;
}
+ ReplicationConfig repConfig = ReplicationConfig.fromProto(
+ type, factor, keyArgs.getEcReplicationConfig());
// TODO: Here we are allocating block with out any check for
// bucket/key/volume or not and also with out any authorization checks.
List< OmKeyLocationInfo > omKeyLocationInfoList =
allocateBlock(ozoneManager.getScmClient(),
- ozoneManager.getBlockTokenSecretManager(), type, factor,
+ ozoneManager.getBlockTokenSecretManager(), repConfig,
new ExcludeList(), requestedSize, scmBlockSize,
ozoneManager.getPreallocateBlocksMax(),
ozoneManager.isGrpcBlockTokenEnabled(),
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
index 18224cd..9ac43c9 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
@@ -25,6 +25,7 @@ import java.util.Map;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
@@ -94,14 +95,15 @@ public class OMAllocateBlockRequest extends OMKeyRequest {
// one uses direct omclient we might be in trouble.
+ ReplicationConfig repConfig = ReplicationConfig.fromProto(keyArgs.getType(),
+ keyArgs.getFactor(), keyArgs.getEcReplicationConfig());
// To allocate atleast one block passing requested size and scmBlockSize
// as same value. When allocating block requested size is same as
// scmBlockSize.
List<OmKeyLocationInfo> omKeyLocationInfoList =
allocateBlock(ozoneManager.getScmClient(),
- ozoneManager.getBlockTokenSecretManager(), keyArgs.getType(),
- keyArgs.getFactor(), excludeList, ozoneManager.getScmBlockSize(),
- ozoneManager.getScmBlockSize(),
+ ozoneManager.getBlockTokenSecretManager(), repConfig, excludeList,
+ ozoneManager.getScmBlockSize(), ozoneManager.getScmBlockSize(),
ozoneManager.getPreallocateBlocksMax(),
ozoneManager.isGrpcBlockTokenEnabled(), ozoneManager.getOMNodeId());
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
index 06c104b..5fdfe6c 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
@@ -27,6 +27,7 @@ import java.util.stream.Collectors;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.om.OMConfigKeys;
@@ -141,6 +142,8 @@ public class OMKeyCreateRequest extends OMKeyRequest {
type = useRatis ? HddsProtos.ReplicationType.RATIS :
HddsProtos.ReplicationType.STAND_ALONE;
}
+ ReplicationConfig repConfig = ReplicationConfig.fromProto(
+ type, factor, keyArgs.getEcReplicationConfig());
// TODO: Here we are allocating block with out any check for
// bucket/key/volume or not and also with out any authorization checks.
@@ -149,7 +152,7 @@ public class OMKeyCreateRequest extends OMKeyRequest {
List<OmKeyLocationInfo> omKeyLocationInfoList =
allocateBlock(ozoneManager.getScmClient(),
- ozoneManager.getBlockTokenSecretManager(), type, factor,
+ ozoneManager.getBlockTokenSecretManager(), repConfig,
new ExcludeList(), requestedSize, scmBlockSize,
ozoneManager.getPreallocateBlocksMax(),
ozoneManager.isGrpcBlockTokenEnabled(),
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
index 0647be0..caa2e80 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
@@ -57,7 +57,6 @@ import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension
.EncryptedKeyVersion;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.hdds.client.BlockID;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
@@ -113,12 +112,9 @@ public abstract class OMKeyRequest extends OMClientRequest {
@SuppressWarnings("parameternumber")
protected List< OmKeyLocationInfo > allocateBlock(ScmClient scmClient,
OzoneBlockTokenSecretManager secretManager,
- HddsProtos.ReplicationType replicationType,
- HddsProtos.ReplicationFactor replicationFactor,
- ExcludeList excludeList, long requestedSize, long scmBlockSize,
- int preallocateBlocksMax, boolean grpcBlockTokenEnabled, String omID)
- throws IOException {
-
+ ReplicationConfig replicationConfig, ExcludeList excludeList,
+ long requestedSize, long scmBlockSize, int preallocateBlocksMax,
+ boolean grpcBlockTokenEnabled, String omID) throws IOException {
int numBlocks = Math.min((int) ((requestedSize - 1) / scmBlockSize + 1),
preallocateBlocksMax);
@@ -127,8 +123,8 @@ public abstract class OMKeyRequest extends OMClientRequest {
List<AllocatedBlock> allocatedBlocks;
try {
allocatedBlocks = scmClient.getBlockClient()
- .allocateBlock(scmBlockSize, numBlocks, replicationType,
- replicationFactor, omID, excludeList);
+ .allocateBlock(scmBlockSize, numBlocks, replicationConfig, omID,
+ excludeList);
} catch (SCMException ex) {
if (ex.getResult()
.equals(SCMException.ResultCodes.SAFE_MODE_EXCEPTION)) {
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
index 22f8fa2..4dfedd2 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
@@ -24,6 +24,7 @@ import java.util.Random;
import java.util.UUID;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.ozone.om.ResolvedBucket;
import org.apache.hadoop.ozone.om.KeyManager;
import org.apache.hadoop.ozone.om.KeyManagerImpl;
@@ -155,8 +156,7 @@ public class TestOMKeyRequest {
allocatedBlocks.add(allocatedBlock);
when(scmBlockLocationProtocol.allocateBlock(anyLong(), anyInt(),
- any(HddsProtos.ReplicationType.class),
- any(HddsProtos.ReplicationFactor.class),
+ any(ReplicationConfig.class),
anyString(), any(ExcludeList.class))).thenReturn(allocatedBlocks);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org