You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ra...@apache.org on 2022/06/03 15:08:00 UTC
[ozone] branch master updated: HDDS-6683. Refactor OM server bucket layout configuration usage (#3477)
This is an automated email from the ASF dual-hosted git repository.
rakeshr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new b7bf8a4260 HDDS-6683. Refactor OM server bucket layout configuration usage (#3477)
b7bf8a4260 is described below
commit b7bf8a42606c980ed1c861895b631278bf08c3fb
Author: tanvipenumudy <46...@users.noreply.github.com>
AuthorDate: Fri Jun 3 20:37:55 2022 +0530
HDDS-6683. Refactor OM server bucket layout configuration usage (#3477)
---
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 12 +----
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 4 +-
.../ozone/om/TestOMStartupWithBucketLayout.java | 8 ++--
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 6 +--
.../hadoop/ozone/shell/TestOzoneShellHA.java | 17 ++++---
.../ozone/shell/TestOzoneShellHAWithFSO.java | 52 ++++++++++++++++++++++
.../om/request/bucket/OMBucketCreateRequest.java | 9 ++--
.../request/bucket/TestOMBucketCreateRequest.java | 17 ++++++-
.../bucket/TestOMBucketCreateRequestWithFSO.java | 3 ++
9 files changed, 99 insertions(+), 29 deletions(-)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java
index 14a73e3b9c..ca55f6a2f4 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java
@@ -68,8 +68,6 @@ public class TestDirectoryDeletingServiceWithFSO {
private static final Logger LOG =
LoggerFactory.getLogger(TestDirectoryDeletingServiceWithFSO.class);
- private static boolean isBucketFSOptimized = true;
- private static boolean enabledFileSystemPaths = true;
private static boolean omRatisEnabled = true;
private static MiniOzoneCluster cluster;
@@ -86,20 +84,14 @@ public class TestDirectoryDeletingServiceWithFSO {
TimeUnit.MILLISECONDS);
conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, omRatisEnabled);
conf.setBoolean(OZONE_ACL_ENABLED, true);
- if (isBucketFSOptimized) {
- conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
- BucketLayout.FILE_SYSTEM_OPTIMIZED.name());
- } else {
- conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS,
- enabledFileSystemPaths);
- }
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(3)
.build();
cluster.waitForClusterToBeReady();
// create a volume and a bucket to be used by OzoneFileSystem
- OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
+ OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster,
+ BucketLayout.FILE_SYSTEM_OPTIMIZED);
volumeName = bucket.getVolumeName();
bucketName = bucket.getName();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java
index 4b1e724566..eb8f69aa93 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java
@@ -59,7 +59,9 @@ public final class TestDataUtil {
BucketArgs omBucketArgs;
BucketArgs.Builder builder = BucketArgs.newBuilder();
builder.setStorageType(StorageType.DISK);
- builder.setBucketLayout(bucketLayout);
+ if (bucketLayout != null) {
+ builder.setBucketLayout(bucketLayout);
+ }
omBucketArgs = builder.build();
return createVolumeAndBucket(cluster, volumeName, bucketName, bucketLayout,
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStartupWithBucketLayout.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStartupWithBucketLayout.java
index e43176021a..214d38647d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStartupWithBucketLayout.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStartupWithBucketLayout.java
@@ -78,10 +78,10 @@ public class TestOMStartupWithBucketLayout {
BucketLayout.FILE_SYSTEM_OPTIMIZED);
verifyBucketLayout(bucket1, BucketLayout.FILE_SYSTEM_OPTIMIZED);
- // 3. verify OM default behavior with LEGACY
+ // 3. verify OM default behavior with empty
restartCluster();
OzoneBucket bucket2 = TestDataUtil.createVolumeAndBucket(cluster,
- BucketLayout.LEGACY);
+ null);
verifyBucketLayout(bucket2, BucketLayout.FILE_SYSTEM_OPTIMIZED);
// 4. create bucket with OBS bucket layout and verify
@@ -122,10 +122,10 @@ public class TestOMStartupWithBucketLayout {
BucketLayout.FILE_SYSTEM_OPTIMIZED);
verifyBucketLayout(bucket1, BucketLayout.FILE_SYSTEM_OPTIMIZED);
- // 3. verify OM default behavior with LEGACY
+ // 3. verify OM default behavior with empty
restartCluster();
OzoneBucket bucket2 = TestDataUtil.createVolumeAndBucket(cluster,
- BucketLayout.LEGACY);
+ null);
verifyBucketLayout(bucket2, BucketLayout.OBJECT_STORE);
// 4. create bucket with OBS bucket layout and verify
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java
index af82b7cab6..c0a844c520 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java
@@ -741,9 +741,9 @@ public class TestObjectStoreWithFSO {
Assert.assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED,
bucket.getBucketLayout());
- // Case 4: Bucket layout: DEFAULT
+ // Case 4: Bucket layout: Empty
sampleBucketName = UUID.randomUUID().toString();
- builder.setBucketLayout(BucketLayout.DEFAULT);
+ builder = BucketArgs.newBuilder();
volume.createBucket(sampleBucketName, builder.build());
bucket = volume.getBucket(sampleBucketName);
Assert.assertEquals(sampleBucketName, bucket.getName());
@@ -756,7 +756,7 @@ public class TestObjectStoreWithFSO {
volume.createBucket(sampleBucketName, builder.build());
bucket = volume.getBucket(sampleBucketName);
Assert.assertEquals(sampleBucketName, bucket.getName());
- Assert.assertNotEquals(BucketLayout.LEGACY, bucket.getBucketLayout());
+ Assert.assertEquals(BucketLayout.LEGACY, bucket.getBucketLayout());
}
private void assertKeyRenamedEx(OzoneBucket bucket, String keyName)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
index d22e4cad48..10df7a8d5b 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
@@ -107,7 +107,6 @@ public class TestOzoneShellHA {
private static File baseDir;
private static File testFile;
private static String testFilePathString;
- private static OzoneConfiguration conf = null;
private static MiniOzoneCluster cluster = null;
private OzoneShell ozoneShell = null;
private OzoneAdmin ozoneAdminShell = null;
@@ -131,8 +130,11 @@ public class TestOzoneShellHA {
*/
@BeforeClass
public static void init() throws Exception {
- conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
+ startCluster(conf);
+ }
+ protected static void startCluster(OzoneConfiguration conf) throws Exception {
String path = GenericTestUtils.getTempPath(
TestOzoneShellHA.class.getSimpleName());
baseDir = new File(path);
@@ -261,7 +263,7 @@ public class TestOzoneShellHA {
}
private String getSetConfStringFromConf(String key) {
- return String.format("--set=%s=%s", key, conf.get(key));
+ return String.format("--set=%s=%s", key, cluster.getConf().get(key));
}
private String generateSetConfString(String key, String value) {
@@ -287,7 +289,7 @@ public class TestOzoneShellHA {
String omNodesKey = ConfUtils.addKeySuffixes(
OMConfigKeys.OZONE_OM_NODES_KEY, omServiceId);
- String omNodesVal = conf.get(omNodesKey);
+ String omNodesVal = cluster.getConf().get(omNodesKey);
res[indexOmNodes] = generateSetConfString(omNodesKey, omNodesVal);
String[] omNodesArr = omNodesVal.split(",");
@@ -392,7 +394,7 @@ public class TestOzoneShellHA {
String omLeaderNodeId = getLeaderOMNodeId();
String omLeaderNodeAddrKey = ConfUtils.addKeySuffixes(
OMConfigKeys.OZONE_OM_ADDRESS_KEY, omServiceId, omLeaderNodeId);
- String omLeaderNodeAddr = conf.get(omLeaderNodeAddrKey);
+ String omLeaderNodeAddr = cluster.getConf().get(omLeaderNodeAddrKey);
String omLeaderNodeAddrWithoutPort = omLeaderNodeAddr.split(":")[0];
// Test case 2: ozone sh volume create o3://om1/volume2
@@ -560,7 +562,8 @@ public class TestOzoneShellHA {
@Test
public void testDeleteToTrashOrSkipTrash() throws Exception {
final String hostPrefix = OZONE_OFS_URI_SCHEME + "://" + omServiceId;
- OzoneConfiguration clientConf = getClientConfForOFS(hostPrefix, conf);
+ OzoneConfiguration clientConf =
+ getClientConfForOFS(hostPrefix, cluster.getConf());
OzoneFsShell shell = new OzoneFsShell(clientConf);
FileSystem fs = FileSystem.get(clientConf);
final String strDir1 = hostPrefix + "/volumed2t/bucket1/dir1";
@@ -634,7 +637,7 @@ public class TestOzoneShellHA {
// (default is TrashPolicyDefault)
final String hostPrefix = OZONE_OFS_URI_SCHEME + "://" + omServiceId;
OzoneConfiguration clientConf =
- getClientConfForOzoneTrashPolicy(hostPrefix, conf);
+ getClientConfForOzoneTrashPolicy(hostPrefix, cluster.getConf());
OzoneFsShell shell = new OzoneFsShell(clientConf);
int res;
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHAWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHAWithFSO.java
new file mode 100644
index 0000000000..fef5624726
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHAWithFSO.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+/**
+ * This class tests Ozone sh shell command with FSO.
+ * Inspired by TestS3Shell
+ */
+public class TestOzoneShellHAWithFSO extends TestOzoneShellHA {
+
+ /**
+ * Create a MiniOzoneCluster for testing with using distributed Ozone
+ * handler type.
+ *
+ * @throws Exception
+ */
+ @BeforeClass
+ public static void init() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
+ OMConfigKeys.OZONE_BUCKET_LAYOUT_FILE_SYSTEM_OPTIMIZED);
+ startCluster(conf);
+ }
+
+ /**
+ * shutdown MiniOzoneCluster.
+ */
+ @AfterClass
+ public static void shutdownCluster() {
+ shutdown();
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java
index 8d31ca9794..320957ad95 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java
@@ -51,7 +51,6 @@ import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketEncryptionInfoProto;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketInfo;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketLayoutProto;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateBucketRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateBucketResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
@@ -152,8 +151,12 @@ public class OMBucketCreateRequest extends OMClientRequest {
OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
getOmRequest());
OmBucketInfo omBucketInfo = null;
- if (bucketInfo.getBucketLayout() == null || bucketInfo.getBucketLayout()
- .equals(BucketLayoutProto.LEGACY)) {
+
+ // bucketInfo.hasBucketLayout() would be true when user sets bucket layout.
+ // Now, OM will create bucket with the user specified bucket layout.
+ // When the value is not specified by the user, OM will use the
+ // "ozone.default.bucket.layout" configured value.
+ if (!bucketInfo.hasBucketLayout()) {
// Bucket Layout argument was not passed during bucket creation.
String omDefaultBucketLayout = ozoneManager.getOMDefaultBucketLayout();
BucketLayout defaultType = BucketLayout.fromString(omDefaultBucketLayout);
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java
index 83b34006a7..f57da71bd2 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java
@@ -22,6 +22,7 @@ package org.apache.hadoop.ozone.om.request.bucket;
import java.util.UUID;
import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.ozone.test.LambdaTestUtils;
import org.junit.Assert;
import org.junit.Test;
@@ -100,7 +101,6 @@ public class TestOMBucketCreateRequest extends TestBucketRequest {
Assert.assertNull(omMetadataManager.getBucketTable().get(bucketKey));
}
-
@Test
public void testValidateAndUpdateCacheWithBucketAlreadyExists()
throws Exception {
@@ -124,6 +124,21 @@ public class TestOMBucketCreateRequest extends TestBucketRequest {
omResponse.getStatus());
}
+ @Test
+ public void testValidateAndUpdateCacheVerifyBucketLayout() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
+
+ OMBucketCreateRequest omBucketCreateRequest = doPreExecute(volumeName,
+ bucketName);
+
+ doValidateAndUpdateCache(volumeName, bucketName,
+ omBucketCreateRequest.getOmRequest());
+
+ Assert.assertEquals(BucketLayout.LEGACY,
+ omMetadataManager.getBucketTable().get(bucketKey).getBucketLayout());
+ }
private OMBucketCreateRequest doPreExecute(String volumeName,
String bucketName) throws Exception {
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java
index 3dbddd7dea..44efbe9ebf 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java
@@ -44,6 +44,7 @@ public class TestOMBucketCreateRequestWithFSO
BucketLayout.FILE_SYSTEM_OPTIMIZED.name());
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
+ String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
Assert.assertEquals(0, omMetrics.getNumFSOBucketCreates());
@@ -54,6 +55,8 @@ public class TestOMBucketCreateRequestWithFSO
omBucketCreateRequest.getOmRequest());
Assert.assertEquals(1, omMetrics.getNumFSOBucketCreates());
+ Assert.assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED,
+ omMetadataManager.getBucketTable().get(bucketKey).getBucketLayout());
}
private OMBucketCreateRequest doPreExecute(String volumeName,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org