You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by um...@apache.org on 2021/10/04 16:11:58 UTC
[ozone] branch HDDS-3816-ec updated: HDDS-5784: EC: In
BasicRootedOzoneClientAdapterImpl,
Inherit bucket default replication config only in the case of EC. (#2698)
This is an automated email from the ASF dual-hosted git repository.
umamahesh pushed a commit to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-3816-ec by this push:
new fc17518 HDDS-5784: EC: In BasicRootedOzoneClientAdapterImpl, Inherit bucket default replication config only in the case of EC. (#2698)
fc17518 is described below
commit fc17518932220806595d97c8f250f8db8488e811
Author: Uma Maheswara Rao G <um...@apache.org>
AuthorDate: Mon Oct 4 09:11:43 2021 -0700
HDDS-5784: EC: In BasicRootedOzoneClientAdapterImpl, Inherit bucket default replication config only in the case of EC. (#2698)
Co-authored-by: Uma Maheswara Rao G <um...@cloudera.com>
---
...OzoneManagerProtocolClientSideTranslatorPB.java | 10 +++-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 69 +++++++++++++++++++++-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 18 ++++--
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 14 ++++-
4 files changed, 99 insertions(+), 12 deletions(-)
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index 7743ab7..6911783 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -1481,9 +1481,13 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
.addAllAcls(args.getAcls().stream().map(a ->
OzoneAcl.toProtobuf(a)).collect(Collectors.toList()));
if (args.getReplicationConfig() != null) {
- keyArgsBuilder.setFactor(
- ReplicationConfig
- .getLegacyFactor(args.getReplicationConfig()));
+ if (args.getReplicationConfig() instanceof ECReplicationConfig) {
+ keyArgsBuilder.setEcReplicationConfig(
+ ((ECReplicationConfig) args.getReplicationConfig()).toProto());
+ } else {
+ keyArgsBuilder.setFactor(
+ ReplicationConfig.getLegacyFactor(args.getReplicationConfig()));
+ }
keyArgsBuilder.setType(args.getReplicationConfig().getReplicationType());
}
CreateFileRequest createFileRequest = CreateFileRequest.newBuilder()
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
index e779f21..4c29404 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
@@ -31,7 +31,12 @@ import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.fs.TrashPolicy;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OFSPath;
import org.apache.hadoop.ozone.OzoneAcl;
@@ -82,6 +87,7 @@ import java.util.Optional;
import java.util.Random;
import java.util.Set;
import java.util.TreeSet;
+import java.util.UUID;
import java.util.stream.Collectors;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_CHECKPOINT_INTERVAL_KEY;
@@ -178,7 +184,7 @@ public class TestRootedOzoneFileSystem {
}
conf.setBoolean(OzoneConfigKeys.OZONE_ACL_ENABLED, enableAcl);
cluster = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(3)
+ .setNumDatanodes(5)
.build();
cluster.waitForClusterToBeReady();
objectStore = cluster.getClient().getObjectStore();
@@ -1512,4 +1518,65 @@ public class TestRootedOzoneFileSystem {
}
}
+ @Test
+ public void testBucketDefaultsShouldNotBeInheritedToFileForNonEC()
+ throws Exception {
+ BucketArgs.Builder builder = BucketArgs.newBuilder();
+ builder.setStorageType(StorageType.DISK);
+ builder.setBucketLayout(BucketLayout.LEGACY);
+ builder.setDefaultReplicationConfig(
+ new DefaultReplicationConfig(ReplicationType.STAND_ALONE,
+ ReplicationFactor.ONE));
+ BucketArgs omBucketArgs = builder.build();
+ String vol = UUID.randomUUID().toString();
+ String buck = UUID.randomUUID().toString();
+ final OzoneBucket bucket100 = TestDataUtil
+ .createVolumeAndBucket(cluster, vol, buck, BucketLayout.LEGACY,
+ omBucketArgs);
+ Assert.assertEquals(ReplicationType.STAND_ALONE.name(),
+ bucket100.getReplicationConfig().getReplicationType().name());
+
+ // Bucket has default STAND_ALONE and client has default RATIS.
+ // In this case, it should not inherit from bucket
+ try (OzoneFSOutputStream file = adapter
+ .createFile(vol + "/" + buck + "/test", (short) 3, true, false)) {
+ file.write(new byte[1024]);
+ }
+ OFSPath ofsPath = new OFSPath(vol + "/" + buck + "/test");
+ final OzoneBucket bucket = adapter.getBucket(ofsPath, false);
+ final OzoneKeyDetails key = bucket.getKey(ofsPath.getKeyName());
+ Assert.assertEquals(key.getReplicationConfig().getReplicationType().name(),
+ ReplicationType.RATIS.name());
+ }
+
+ @Test
+ public void testBucketDefaultsShouldBeInheritedToFileForEC()
+ throws Exception {
+ BucketArgs.Builder builder = BucketArgs.newBuilder();
+ builder.setStorageType(StorageType.DISK);
+ builder.setBucketLayout(BucketLayout.LEGACY);
+ builder.setDefaultReplicationConfig(
+ new DefaultReplicationConfig(ReplicationType.EC,
+ new ECReplicationConfig(3, 2)));
+ BucketArgs omBucketArgs = builder.build();
+ String vol = UUID.randomUUID().toString();
+ String buck = UUID.randomUUID().toString();
+ final OzoneBucket bucket101 = TestDataUtil
+ .createVolumeAndBucket(cluster, vol, buck, BucketLayout.LEGACY,
+ omBucketArgs);
+ Assert.assertEquals(ReplicationType.EC.name(),
+ bucket101.getReplicationConfig().getReplicationType().name());
+ // Bucket has default EC and client has default RATIS.
+ // In this case, it should inherit from bucket
+ try (OzoneFSOutputStream file = adapter
+ .createFile(vol + "/" + buck + "/test", (short) 3, true, false)) {
+ file.write(new byte[1024]);
+ }
+ OFSPath ofsPath = new OFSPath(vol + "/" + buck + "/test");
+ final OzoneBucket bucket = adapter.getBucket(ofsPath, false);
+ final OzoneKeyDetails key = bucket.getKey(ofsPath.getKeyName());
+ Assert.assertEquals(key.getReplicationConfig().getReplicationType().name(),
+ ReplicationType.EC.name());
+ }
+
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java
index 027afa3..5c41cd7 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java
@@ -56,6 +56,19 @@ public final class TestDataUtil {
public static OzoneBucket createVolumeAndBucket(MiniOzoneCluster cluster,
String volumeName, String bucketName, BucketLayout bucketLayout)
throws IOException {
+ BucketArgs omBucketArgs;
+ BucketArgs.Builder builder = BucketArgs.newBuilder();
+ builder.setStorageType(StorageType.DISK);
+ builder.setBucketLayout(bucketLayout);
+ omBucketArgs = builder.build();
+
+ return createVolumeAndBucket(cluster, volumeName, bucketName, bucketLayout,
+ omBucketArgs);
+ }
+
+ public static OzoneBucket createVolumeAndBucket(MiniOzoneCluster cluster,
+ String volumeName, String bucketName, BucketLayout bucketLayout,
+ BucketArgs omBucketArgs) throws IOException {
String userName = "user" + RandomStringUtils.randomNumeric(5);
String adminName = "admin" + RandomStringUtils.randomNumeric(5);
@@ -69,11 +82,6 @@ public final class TestDataUtil {
objectStore.createVolume(volumeName, volumeArgs);
OzoneVolume volume = objectStore.getVolume(volumeName);
- BucketArgs omBucketArgs;
- BucketArgs.Builder builder = BucketArgs.newBuilder();
- builder.setStorageType(StorageType.DISK);
- builder.setBucketLayout(bucketLayout);
- omBucketArgs = builder.build();
volume.createBucket(bucketName, omBucketArgs);
return volume.getBucket(bucketName);
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
index ee17992..3829878 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
@@ -41,11 +41,13 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ozone.OFSPath;
@@ -315,9 +317,15 @@ public class BasicRootedOzoneClientAdapterImpl
try {
// Hadoop CopyCommands class always sets recursive to true
OzoneBucket bucket = getBucket(ofsPath, recursive);
- ReplicationConfig replConfig = this.replicationConfig;
- // Since the bucket has the right default replication, we are using it.
- if (bucket.getReplicationConfig() != null) {
+ // if client side replication config is null, we will take legacy default
+ // value at client that is RATIS.
+ ReplicationConfig replConfig = this.replicationConfig != null ?
+ this.replicationConfig :
+ new RatisReplicationConfig(HddsProtos.ReplicationFactor.THREE);
+ // Since the bucket has the default replication and type is EC. So, we are
+ // using it.
+ if (bucket.getReplicationConfig() != null && bucket.getReplicationConfig()
+ .getReplicationType() == HddsProtos.ReplicationType.EC) {
replConfig = bucket.getReplicationConfig();
}
OzoneOutputStream ozoneOutputStream = null;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org