You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by um...@apache.org on 2021/06/25 17:24:36 UTC
[ozone] branch HDDS-3816-ec updated: HDDS-5343 : EC: Add
ECReplicationConfig into KeyInfo proto. (#2340). Contributed by Uma
Maheswara Rao G.
This is an automated email from the ASF dual-hosted git repository.
umamahesh pushed a commit to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-3816-ec by this push:
new f53a93e HDDS-5343 : EC: Add ECReplicationConfig into KeyInfo proto. (#2340). Contributed by Uma Maheswara Rao G.
f53a93e is described below
commit f53a93e6a9cfa2018882aa2672e5223c4f4f9703
Author: Uma Maheswara Rao G <um...@apache.org>
AuthorDate: Fri Jun 25 10:24:17 2021 -0700
HDDS-5343 : EC: Add ECReplicationConfig into KeyInfo proto. (#2340). Contributed by Uma Maheswara Rao G.
---
.../hadoop/ozone/client/TestOzoneClient.java | 19 +++++++
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 15 +++--
...OzoneManagerProtocolClientSideTranslatorPB.java | 28 +++++++--
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 66 +++++++++++++++++++---
.../src/main/proto/OmClientProtocol.proto | 4 +-
5 files changed, 113 insertions(+), 19 deletions(-)
diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneClient.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneClient.java
index 2f4ebcf..e43401f 100644
--- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneClient.java
+++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneClient.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.client;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.InMemoryConfiguration;
@@ -183,6 +184,24 @@ public class TestOzoneClient {
}
}
+ @Test
+ public void testPutKeyWithECReplicationConfig() throws IOException {
+ String value = new String(new byte[1024], UTF_8);
+ OzoneBucket bucket = getOzoneBucket();
+
+ for (int i = 0; i < 10; i++) {
+ String keyName = UUID.randomUUID().toString();
+ try (OzoneOutputStream out = bucket
+ .createKey(keyName, value.getBytes(UTF_8).length,
+ new ECReplicationConfig(3, 2), new HashMap<>())) {
+ out.write(value.getBytes(UTF_8));
+ out.write(value.getBytes(UTF_8));
+ }
+ OzoneKey key = bucket.getKey(keyName);
+ Assert.assertEquals(keyName, key.getName());
+ }
+ }
+
private OzoneBucket getOzoneBucket() throws IOException {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
index 996e04f..9013fe2 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
@@ -29,6 +29,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.client.ContainerBlockID;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.OzoneConsts;
@@ -521,9 +522,14 @@ public final class OmKeyInfo extends WithParentObjectId {
.setVolumeName(volumeName)
.setBucketName(bucketName)
.setDataSize(dataSize)
- .setType(replicationConfig.getReplicationType())
- .setFactor(ReplicationConfig.getLegacyFactor(replicationConfig))
- .setLatestVersion(latestVersion)
+ .setType(replicationConfig.getReplicationType());
+ if (replicationConfig instanceof ECReplicationConfig) {
+ kb.setEcReplicationConfig(
+ ((ECReplicationConfig) replicationConfig).toProto());
+ } else {
+ kb.setFactor(ReplicationConfig.getLegacyFactor(replicationConfig));
+ }
+ kb.setLatestVersion(latestVersion)
.addAllKeyLocationList(keyLocations)
.setCreationTime(creationTime)
.setModificationTime(modificationTime)
@@ -563,7 +569,8 @@ public final class OmKeyInfo extends WithParentObjectId {
.setCreationTime(keyInfo.getCreationTime())
.setModificationTime(keyInfo.getModificationTime())
.setReplicationConfig(ReplicationConfig
- .fromTypeAndFactor(keyInfo.getType(), keyInfo.getFactor()))
+ .fromProto(keyInfo.getType(), keyInfo.getFactor(),
+ keyInfo.getEcReplicationConfig()))
.addAllMetadata(KeyValueUtil.getFromProtobuf(keyInfo.getMetadataList()))
.setFileEncryptionInfo(keyInfo.hasFileEncryptionInfo() ?
OMPBHelper.convert(keyInfo.getFileEncryptionInfo()) : null)
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index c13ef9f..28aa93d 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -25,6 +25,7 @@ import java.util.Map;
import java.util.stream.Collectors;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
import org.apache.hadoop.hdds.tracing.TracingUtil;
@@ -560,8 +561,13 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
}
if (args.getReplicationConfig() != null) {
- keyArgs.setFactor(
- ReplicationConfig.getLegacyFactor(args.getReplicationConfig()));
+ if (args.getReplicationConfig() instanceof ECReplicationConfig) {
+ keyArgs.setEcReplicationConfig(
+ ((ECReplicationConfig) args.getReplicationConfig()).toProto());
+ } else {
+ keyArgs.setFactor(
+ ReplicationConfig.getLegacyFactor(args.getReplicationConfig()));
+ }
keyArgs.setType(args.getReplicationConfig().getReplicationType());
}
@@ -618,8 +624,13 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
.setDataSize(args.getDataSize());
if (args.getReplicationConfig() != null) {
- keyArgs.setFactor(
- ReplicationConfig.getLegacyFactor(args.getReplicationConfig()));
+ if (args.getReplicationConfig() instanceof ECReplicationConfig) {
+ keyArgs.setEcReplicationConfig(
+ ((ECReplicationConfig) args.getReplicationConfig()).toProto());
+ } else {
+ keyArgs.setFactor(
+ ReplicationConfig.getLegacyFactor(args.getReplicationConfig()));
+ }
keyArgs.setType(args.getReplicationConfig().getReplicationType());
}
@@ -653,8 +664,13 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
.collect(Collectors.toList()));
if (args.getReplicationConfig() != null) {
- keyArgsBuilder.setFactor(
- ReplicationConfig.getLegacyFactor(args.getReplicationConfig()));
+ if (args.getReplicationConfig() instanceof ECReplicationConfig) {
+ keyArgsBuilder.setEcReplicationConfig(
+ ((ECReplicationConfig) args.getReplicationConfig()).toProto());
+ } else {
+ keyArgsBuilder.setFactor(
+ ReplicationConfig.getLegacyFactor(args.getReplicationConfig()));
+ }
keyArgsBuilder.setType(args.getReplicationConfig().getReplicationType());
}
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfo.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfo.java
index eaf83d9..6b005c1 100644
--- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfo.java
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfo.java
@@ -18,13 +18,16 @@
package org.apache.hadoop.ozone.om.helpers;
import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo.Builder;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.apache.hadoop.util.Time;
import org.junit.Assert;
@@ -35,6 +38,9 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.EC;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.RATIS;
import static org.apache.hadoop.ozone.ClientVersions.CURRENT_VERSION;
import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS;
@@ -45,23 +51,67 @@ public class TestOmKeyInfo {
@Test
public void protobufConversion() {
- OmKeyInfo key = new Builder()
+ OmKeyInfo key =
+ createOmKeyInfo(new RatisReplicationConfig(ReplicationFactor.THREE));
+
+ OmKeyInfo keyAfterSerialization =
+ OmKeyInfo.getFromProtobuf(key.getProtobuf(CURRENT_VERSION));
+
+ Assert.assertEquals(key, keyAfterSerialization);
+ }
+
+ @Test
+ public void getProtobufMessageEC() {
+ OmKeyInfo key =
+ createOmKeyInfo(new RatisReplicationConfig(ReplicationFactor.THREE));
+ OzoneManagerProtocolProtos.KeyInfo omKeyProto =
+ key.getProtobuf(CURRENT_VERSION);
+
+ // No EC Config
+ Assert.assertFalse(omKeyProto.hasEcReplicationConfig());
+ Assert.assertEquals(THREE, omKeyProto.getFactor());
+ Assert.assertEquals(RATIS, omKeyProto.getType());
+
+ // Reconstruct object from Proto
+ OmKeyInfo recovered = OmKeyInfo.getFromProtobuf(omKeyProto);
+ Assert.assertEquals(RATIS,
+ recovered.getReplicationConfig().getReplicationType());
+ Assert.assertTrue(
+ recovered.getReplicationConfig() instanceof RatisReplicationConfig);
+
+ // EC Config
+ key = createOmKeyInfo(new ECReplicationConfig(3, 2));
+ omKeyProto = key.getProtobuf(CURRENT_VERSION);
+
+ Assert.assertEquals(3, omKeyProto.getEcReplicationConfig().getData());
+ Assert.assertEquals(2, omKeyProto.getEcReplicationConfig().getParity());
+ Assert.assertFalse(omKeyProto.hasFactor());
+ Assert.assertEquals(EC, omKeyProto.getType());
+
+ // Reconstruct object from Proto
+ recovered = OmKeyInfo.getFromProtobuf(omKeyProto);
+ Assert.assertEquals(EC,
+ recovered.getReplicationConfig().getReplicationType());
+ Assert.assertTrue(
+ recovered.getReplicationConfig() instanceof ECReplicationConfig);
+ ECReplicationConfig config =
+ (ECReplicationConfig) recovered.getReplicationConfig();
+ Assert.assertEquals(3, config.getData());
+ Assert.assertEquals(2, config.getParity());
+ }
+
+ private OmKeyInfo createOmKeyInfo(ReplicationConfig replicationConfig) {
+ return new Builder()
.setKeyName("key1")
.setBucketName("bucket")
.setVolumeName("vol1")
.setCreationTime(123L)
.setModificationTime(123L)
.setDataSize(123L)
- .setReplicationConfig(
- new RatisReplicationConfig(ReplicationFactor.THREE))
+ .setReplicationConfig(replicationConfig)
.addMetadata("key1", "value1")
.addMetadata("key2", "value2")
.build();
-
- OmKeyInfo keyAfterSerialization =
- OmKeyInfo.getFromProtobuf(key.getProtobuf(CURRENT_VERSION));
-
- Assert.assertEquals(key, keyAfterSerialization);
}
@Test
diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 8c46e3f..07c6767 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -746,6 +746,7 @@ message KeyArgs {
// This will be set when user performs delete directory recursively.
optional bool recursive = 17;
+ optional hadoop.hdds.ECReplicationConfig ecReplicationConfig = 18;
}
message KeyLocation {
@@ -779,7 +780,7 @@ message KeyInfo {
required string keyName = 3;
required uint64 dataSize = 4;
required hadoop.hdds.ReplicationType type = 5;
- required hadoop.hdds.ReplicationFactor factor = 6;
+ optional hadoop.hdds.ReplicationFactor factor = 6;
repeated KeyLocationList keyLocationList = 7;
required uint64 creationTime = 8;
required uint64 modificationTime = 9;
@@ -790,6 +791,7 @@ message KeyInfo {
optional uint64 objectID = 14;
optional uint64 updateID = 15;
optional uint64 parentID = 16;
+ optional hadoop.hdds.ECReplicationConfig ecReplicationConfig = 17;
}
message DirectoryInfo {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org