You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ra...@apache.org on 2021/10/09 17:15:14 UTC
[ozone] branch master updated: HDDS-5373. [FSO] Define default
bucket layout. (#2707)
This is an automated email from the ASF dual-hosted git repository.
rakeshr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 3e05e71 HDDS-5373. [FSO] Define default bucket layout. (#2707)
3e05e71 is described below
commit 3e05e7194248ac8cbf6e8d5f79791369e59667d4
Author: Jyotinder Singh <jy...@gmail.com>
AuthorDate: Sat Oct 9 22:44:50 2021 +0530
HDDS-5373. [FSO] Define default bucket layout. (#2707)
---
.../common/src/main/resources/ozone-default.xml | 19 +++-
hadoop-hdds/docs/content/feature/PrefixFSO.md | 16 ++++
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 10 ++
.../hadoop/ozone/om/helpers/BucketLayout.java | 8 ++
.../apache/hadoop/ozone/om/TestObjectStore.java | 103 +++++++++++++++++++++
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 3 +-
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 6 +-
.../hadoop/ozone/shell/TestNSSummaryAdmin.java | 6 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 16 ++++
.../org/apache/hadoop/ozone/om/OzoneManager.java | 26 +++++-
.../om/request/bucket/OMBucketCreateRequest.java | 11 +--
.../ozone/om/request/TestOMRequestUtils.java | 20 ++--
.../bucket/TestOMBucketCreateRequestWithFSO.java | 5 +
.../ozone/shell/bucket/CreateBucketHandler.java | 6 +-
14 files changed, 229 insertions(+), 26 deletions(-)
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index ce04904..987f94b 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -2108,7 +2108,7 @@
not be renewed.
</description>
</property>
-
+
<property>
<name>ozone.fs.iterate.batch-size</name>
<value>100</value>
@@ -2813,7 +2813,7 @@
<tag>OZONE, SCM</tag>
<description>Thread pool size configured to process container reports.
</description>
- </property>
+ </property>
<property>
<name>ozone.scm.datanode.ratis.volume.free-space.min</name>
@@ -2988,4 +2988,19 @@
OM/SCM/DN/S3GATEWAY Server connection timeout in milliseconds.
</description>
</property>
+
+ <property>
+ <name>ozone.default.bucket.layout</name>
+ <value>OBJECT_STORE</value>
+ <tag>OZONE, MANAGEMENT</tag>
+ <description>
+ Default bucket layout used by Ozone Manager during bucket creation when a client does not specify the
+ bucket layout option. Supported values are OBJECT_STORE and FILE_SYSTEM_OPTIMIZED.
+ OBJECT_STORE: This layout allows the bucket to behave as a pure object store and will not allow
+ interoperability between S3 and FS APIs.
+ FILE_SYSTEM_OPTIMIZED: This layout allows the bucket to support atomic rename/delete operations and
+ also allows interoperability between S3 and FS APIs. Keys written via S3 API with a "/" delimiter
+ will create intermediate directories.
+ </description>
+ </property>
</configuration>
diff --git a/hadoop-hdds/docs/content/feature/PrefixFSO.md b/hadoop-hdds/docs/content/feature/PrefixFSO.md
index 130f284..f54deea 100644
--- a/hadoop-hdds/docs/content/feature/PrefixFSO.md
+++ b/hadoop-hdds/docs/content/feature/PrefixFSO.md
@@ -72,4 +72,20 @@ By default the feature is disabled. It can be enabled with the following
<name>ozone.om.metadata.layout</name>
<value>PREFIX</value>
</property>
+```
+
+In reference to efforts towards supporting protocol aware buckets
+within an Ozone cluster, the following configuration can be used
+to define the default value for bucket layout during bucket creation
+if the client has not specified the bucket layout argument.
+Supported values are `OBJECT_STORE` and `FILE_SYSTEM_OPTIMIZED`.
+
+By default, the buckets will default to `OBJECT_STORE` behaviour.
+
+```XML
+
+<property>
+ <name>ozone.default.bucket.layout</name>
+ <value>OBJECT_STORE</value>
+</property>
```
\ No newline at end of file
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index a02484d..0b6f325 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.om;
import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.ratis.util.TimeDuration;
/**
@@ -270,6 +271,15 @@ public final class OMConfigKeys {
public static final String OZONE_OM_METADATA_LAYOUT_PREFIX = "PREFIX";
+ // Default bucket layout used by Ozone Manager during bucket creation
+ // when a client does not specify the bucket layout option.
+ public static final String OZONE_DEFAULT_BUCKET_LAYOUT =
+ "ozone.default.bucket.layout";
+ public static final String OZONE_DEFAULT_BUCKET_LAYOUT_DEFAULT =
+ BucketLayout.OBJECT_STORE.name();
+ public static final String OZONE_DEFAULT_BUCKET_LAYOUT_FILE_SYSTEM_OPTIMIZED =
+ BucketLayout.FILE_SYSTEM_OPTIMIZED.name();
+
/**
* Configuration properties for Directory Deleting Service.
*/
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketLayout.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketLayout.java
index 91959c9..7705186 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketLayout.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketLayout.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.om.helpers;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
/**
@@ -59,4 +60,11 @@ public enum BucketLayout {
"Error: BucketLayout not found, type=" + this);
}
}
+
+ public static BucketLayout fromString(String value) {
+ // This will never be null in production but can be null in mocked
+ // unit test cases.
+ // Added safer `isBlank` check for unit test cases.
+ return StringUtils.isBlank(value) ? LEGACY : BucketLayout.valueOf(value);
+ }
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStore.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStore.java
new file mode 100644
index 0000000..8432d8c
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStore.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.om;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.client.*;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.junit.*;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.UUID;
+
+/**
+ * Tests to verify Object store without prefix enabled.
+ */
+public class TestObjectStore {
+ private static MiniOzoneCluster cluster = null;
+ private static OzoneConfiguration conf;
+ private static String clusterId;
+ private static String scmId;
+ private static String omId;
+
+ @Rule
+ public Timeout timeout = new Timeout(1200000);
+
+ /**
+ * Create a MiniOzoneCluster for testing.
+ * <p>
+ *
+ * @throws IOException
+ */
+ @BeforeClass
+ public static void init() throws Exception {
+ conf = new OzoneConfiguration();
+ clusterId = UUID.randomUUID().toString();
+ scmId = UUID.randomUUID().toString();
+ omId = UUID.randomUUID().toString();
+ cluster = MiniOzoneCluster.newBuilder(conf).setClusterId(clusterId)
+ .setScmId(scmId).setOmId(omId).build();
+ cluster.waitForClusterToBeReady();
+ }
+
+ /**
+ * Shutdown MiniOzoneCluster.
+ */
+ @AfterClass
+ public static void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testCreateBucketWithBucketLayout() throws Exception {
+ String sampleVolumeName = UUID.randomUUID().toString();
+ String sampleBucketName = UUID.randomUUID().toString();
+ OzoneClient client = cluster.getClient();
+ ObjectStore store = client.getObjectStore();
+ store.createVolume(sampleVolumeName);
+ OzoneVolume volume = store.getVolume(sampleVolumeName);
+
+ // Case 1: Bucket layout: Empty and OM default bucket layout: OBJECT_STORE
+ BucketArgs.Builder builder = BucketArgs.newBuilder();
+ volume.createBucket(sampleBucketName, builder.build());
+ OzoneBucket bucket = volume.getBucket(sampleBucketName);
+ Assert.assertEquals(sampleBucketName, bucket.getName());
+ Assert.assertEquals(BucketLayout.OBJECT_STORE,
+ bucket.getBucketLayout());
+
+ // Case 2: Bucket layout: DEFAULT
+ sampleBucketName = UUID.randomUUID().toString();
+ builder.setBucketLayout(BucketLayout.DEFAULT);
+ volume.createBucket(sampleBucketName, builder.build());
+ bucket = volume.getBucket(sampleBucketName);
+ Assert.assertEquals(sampleBucketName, bucket.getName());
+ Assert.assertEquals(BucketLayout.OBJECT_STORE,
+ bucket.getBucketLayout());
+
+ // Case 3: Bucket layout: LEGACY
+ sampleBucketName = UUID.randomUUID().toString();
+ builder.setBucketLayout(BucketLayout.LEGACY);
+ volume.createBucket(sampleBucketName, builder.build());
+ bucket = volume.getBucket(sampleBucketName);
+ Assert.assertEquals(sampleBucketName, bucket.getName());
+ Assert.assertNotEquals(BucketLayout.LEGACY, bucket.getBucketLayout());
+ }
+}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java
index c16c2c3..ed7a198 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java
@@ -649,7 +649,8 @@ public class TestObjectStoreWithFSO {
Assert.assertEquals(sampleBucketName, bucket.getName());
Assert.assertEquals(BucketLayout.OBJECT_STORE, bucket.getBucketLayout());
- // Case 3: Bucket layout: Empty and OM Metadata layout: PREFIX
+ // Case 3: Bucket layout: Empty and
+ // OM default bucket layout: FILE_SYSTEM_OPTIMIZED
builder = BucketArgs.newBuilder();
sampleBucketName = UUID.randomUUID().toString();
volume.createBucket(sampleBucketName, builder.build());
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java
index c0a9293..76f79e2 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java
@@ -30,6 +30,8 @@ import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
import org.apache.hadoop.ozone.container.TestHelper;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.request.TestOMRequestUtils;
import org.apache.hadoop.ozone.recon.api.NSSummaryEndpoint;
import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse;
import org.apache.hadoop.ozone.recon.api.types.EntityType;
@@ -63,8 +65,8 @@ public class TestReconWithOzoneManagerFSO {
@BeforeClass
public static void init() throws Exception {
conf = new OzoneConfiguration();
- conf.setBoolean("ozone.om.enable.filesystem.paths", true);
- conf.set("ozone.om.metadata.layout", "PREFIX");
+ TestOMRequestUtils.configureFSOptimizedPaths(conf, true,
+ OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX);
cluster =
MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(1)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java
index 1cfd3f7..3b865f6 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.ozone.shell;
import org.apache.hadoop.hdds.cli.OzoneAdmin;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.request.TestOMRequestUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -39,8 +41,8 @@ public class TestNSSummaryAdmin {
@BeforeClass
public static void init() throws Exception {
conf = new OzoneConfiguration();
- conf.setBoolean("ozone.om.enable.filesystem.paths", true);
- conf.set("ozone.om.metadata.layout", "PREFIX");
+ TestOMRequestUtils.configureFSOptimizedPaths(conf, true,
+ OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX);
conf.set(OZONE_RECON_ADDRESS_KEY, "localhost:9888");
cluster = MiniOzoneCluster.newBuilder(conf)
.withoutDatanodes().includeRecon(true).build();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
index a8b1980..c2e8d70 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
@@ -943,4 +943,20 @@ public class TestOzoneShellHA {
testVolumes.forEach(vol -> execute(ozoneShell, new String[] {
"volume", "delete", vol}));
}
+
+ @Test
+ public void testClientBucketLayoutValidation() throws Exception {
+ String[] args = new String[]{
+ "bucket", "create", "o3://" + omServiceId + "/volume7" + "/bucketTest1",
+ "--type", "LEGACY"
+ };
+ try {
+ execute(ozoneShell, args);
+ Assert.fail("Should throw exception on unsupported bucket layouts!");
+ } catch (Exception e) {
+ GenericTestUtils.assertExceptionContains(
+ "expected one of [FILE_SYSTEM_OPTIMIZED, OBJECT_STORE] ",
+ e);
+ }
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 734c0b6..cb554f5 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.conf.ConfigurationException;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
@@ -67,6 +68,7 @@ import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslator
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.hdds.scm.ha.SCMNodeInfo;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
@@ -244,6 +246,8 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_USER_MAX_VOLUME;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_USER_MAX_VOLUME_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_VOLUME_LISTALL_ALLOWED;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_VOLUME_LISTALL_ALLOWED_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT_DEFAULT;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.DETECTED_LOOP_IN_BUCKET_LINKS;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_AUTH_METHOD;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST;
@@ -358,6 +362,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
private final int preallocateBlocksMax;
private final boolean grpcBlockTokenEnabled;
private final boolean useRatisForReplication;
+ private final String defaultBucketLayout;
private boolean isNativeAuthorizerEnabled;
@@ -434,7 +439,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
ResultCodes.OM_NOT_INITIALIZED);
}
omMetaDir = OMStorage.getOmDbDir(configuration);
-
+
this.isSpnegoEnabled = conf.get(OZONE_OM_HTTP_AUTH_TYPE, "simple")
.equals("kerberos");
this.scmBlockSize = (long) conf.getStorageSize(OZONE_SCM_BLOCK_SIZE,
@@ -453,6 +458,21 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY,
OMConfigKeys.OZONE_OM_RATIS_ENABLE_DEFAULT);
+ this.defaultBucketLayout =
+ configuration.getTrimmed(OZONE_DEFAULT_BUCKET_LAYOUT,
+ OZONE_DEFAULT_BUCKET_LAYOUT_DEFAULT);
+ // Make sure defaultBucketLayout is set to a valid value
+ if (!defaultBucketLayout.equals(
+ BucketLayout.FILE_SYSTEM_OPTIMIZED.name()) &&
+ !defaultBucketLayout.equals(BucketLayout.OBJECT_STORE.name())
+ ) {
+ throw new ConfigurationException(
+ defaultBucketLayout +
+ " is not a valid default bucket layout. Supported values are " +
+ BucketLayout.FILE_SYSTEM_OPTIMIZED.name() + ", " +
+ BucketLayout.OBJECT_STORE.name() + ".");
+ }
+
InetSocketAddress omNodeRpcAddr = omNodeDetails.getRpcAddress();
omRpcAddressTxt = new Text(omNodeDetails.getRpcAddressString());
@@ -4072,6 +4092,10 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
.getTrimmed(OZONE_OM_METADATA_LAYOUT, OZONE_OM_METADATA_LAYOUT_DEFAULT);
}
+ public String getOMDefaultBucketLayout() {
+ return this.defaultBucketLayout;
+ }
+
/**
* Create volume which is required for S3Gateway operations.
* @throws IOException
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java
index 84a1b9c..234b422 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.ozone.om.request.bucket;
import com.google.common.base.Optional;
-import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.crypto.CipherSuite;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
@@ -31,7 +30,6 @@ import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.audit.AuditLogger;
import org.apache.hadoop.ozone.audit.OMAction;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.OzoneManager;
@@ -145,12 +143,9 @@ public class OMBucketCreateRequest extends OMClientRequest {
OmBucketInfo omBucketInfo = null;
if (bucketInfo.getBucketLayout() == null || bucketInfo.getBucketLayout()
.equals(BucketLayoutProto.LEGACY)) {
- BucketLayout defaultType = BucketLayout.LEGACY;
- if (StringUtils
- .equalsIgnoreCase(OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX,
- omLayout)) {
- defaultType = BucketLayout.FILE_SYSTEM_OPTIMIZED;
- }
+ // Bucket Layout argument was not passed during bucket creation.
+ String omDefaultBucketLayout = ozoneManager.getOMDefaultBucketLayout();
+ BucketLayout defaultType = BucketLayout.fromString(omDefaultBucketLayout);
omBucketInfo = OmBucketInfo.getFromProtobuf(bucketInfo, defaultType);
} else {
omBucketInfo = OmBucketInfo.getFromProtobuf(bucketInfo);
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java
index 5c40e37..755e06b 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java
@@ -503,12 +503,15 @@ public final class TestOMRequestUtils {
String bucketName, String volumeName, boolean isVersionEnabled,
OzoneManagerProtocolProtos.StorageTypeProto storageTypeProto) {
OzoneManagerProtocolProtos.BucketInfo bucketInfo =
- OzoneManagerProtocolProtos.BucketInfo.newBuilder()
- .setBucketName(bucketName)
- .setVolumeName(volumeName)
- .setIsVersionEnabled(isVersionEnabled)
- .setStorageType(storageTypeProto)
- .addAllMetadata(getMetadataListFSO()).build();
+ OzoneManagerProtocolProtos.BucketInfo.newBuilder()
+ .setBucketName(bucketName)
+ .setVolumeName(volumeName)
+ .setIsVersionEnabled(isVersionEnabled)
+ .setStorageType(storageTypeProto)
+ .addAllMetadata(getMetadataListFSO()).setBucketLayout(
+ OzoneManagerProtocolProtos.BucketLayoutProto.
+ FILE_SYSTEM_OPTIMIZED)
+ .build();
OzoneManagerProtocolProtos.CreateBucketRequest.Builder req =
OzoneManagerProtocolProtos.CreateBucketRequest.newBuilder();
req.setBucketInfo(bucketInfo);
@@ -1036,6 +1039,11 @@ public final class TestOMRequestUtils {
conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS,
enableFileSystemPaths);
conf.set(OMConfigKeys.OZONE_OM_METADATA_LAYOUT, version);
+ if (StringUtils.equalsIgnoreCase(
+ OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX, version)) {
+ conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
+ BucketLayout.FILE_SYSTEM_OPTIMIZED.name());
+ }
}
public static BucketLayout getBucketLayout() {
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java
index f63ce17..a460dc8 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java
@@ -20,6 +20,7 @@
package org.apache.hadoop.ozone.om.request.bucket;
import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.request.TestOMRequestUtils;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
@@ -42,6 +43,8 @@ public class TestOMBucketCreateRequestWithFSO
public void testValidateAndUpdateCacheWithFSO() throws Exception {
when(ozoneManager.getOMMetadataLayout()).thenReturn(
OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX);
+ when(ozoneManager.getOMDefaultBucketLayout()).thenReturn(
+ BucketLayout.FILE_SYSTEM_OPTIMIZED.name());
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
@@ -107,6 +110,8 @@ public class TestOMBucketCreateRequestWithFSO
dbBucketInfo.getMetadata());
Assert.assertEquals(bucketInfoFromProto.getEncryptionKeyInfo(),
dbBucketInfo.getEncryptionKeyInfo());
+ Assert.assertEquals(bucketInfoFromProto.getBucketLayout(),
+ dbBucketInfo.getBucketLayout());
// verify OMResponse.
verifySuccessCreateBucketResponse(omClientResponse.getOMResponse());
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java
index 34cd876..0768219 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java
@@ -51,13 +51,11 @@ public class CreateBucketHandler extends BucketHandler {
"false/unspecified indicates otherwise")
private Boolean isGdprEnforced;
- // TODO: LEGACY should be removed and should not be exposed to the end user
- // we will revisit during the client side defaulting behaviour
- enum AllowedBucketLayouts { FILE_SYSTEM_OPTIMIZED, OBJECT_STORE, LEGACY }
+ enum AllowedBucketLayouts {FILE_SYSTEM_OPTIMIZED, OBJECT_STORE}
@Option(names = { "--type", "-t" },
description = "Allowed Bucket Types: ${COMPLETION-CANDIDATES}",
- defaultValue = "LEGACY")
+ defaultValue = "OBJECT_STORE")
private AllowedBucketLayouts allowedBucketLayout;
@CommandLine.Mixin
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org