You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ra...@apache.org on 2021/12/08 15:32:36 UTC
[ozone] branch master updated: HDDS-5385. [FSO] Remove ozone.om.metadata.layout config in OM (#2887)
This is an automated email from the ASF dual-hosted git repository.
rakeshr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 176ef94 HDDS-5385. [FSO] Remove ozone.om.metadata.layout config in OM (#2887)
176ef94 is described below
commit 176ef94d99f81faa50627004b1cfcfe50f1d46a6
Author: Jyotinder Singh <jy...@gmail.com>
AuthorDate: Wed Dec 8 21:02:17 2021 +0530
HDDS-5385. [FSO] Remove ozone.om.metadata.layout config in OM (#2887)
---
.../common/src/main/resources/ozone-default.xml | 12 ---
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 6 --
.../hadoop/fs/ozone/contract/OzoneContract.java | 2 -
.../hadoop/ozone/TestStandardOutputUtil.java | 84 +++++++++++++++
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 4 +-
.../hadoop/ozone/client/rpc/TestReadRetries.java | 14 +--
.../freon/TestHadoopDirTreeGeneratorWithFSO.java | 4 +-
.../hadoop/ozone/om/TestRecursiveAclWithFSO.java | 3 +-
.../hadoop/ozone/shell/TestNSSummaryAdmin.java | 120 +++++++++++++++++++--
.../org/apache/hadoop/ozone/om/OzoneManager.java | 24 -----
.../om/request/bucket/OMBucketCreateRequest.java | 1 -
.../ozone/om/request/TestOMRequestUtils.java | 20 ++--
.../ozone/om/request/bucket/TestBucketRequest.java | 1 -
.../bucket/TestOMBucketCreateRequestWithFSO.java | 3 -
.../file/TestOMDirectoryCreateRequestWithFSO.java | 3 +-
.../ozone/admin/nssummary/DiskUsageSubCommand.java | 2 +-
.../admin/nssummary/FileSizeDistSubCommand.java | 2 +-
.../ozone/admin/nssummary/NSSummaryAdmin.java | 36 ++++++-
.../ozone/admin/nssummary/NSSummaryCLIUtils.java | 8 +-
.../admin/nssummary/QuotaUsageSubCommand.java | 2 +-
.../ozone/admin/nssummary/SummarySubCommand.java | 2 +-
21 files changed, 256 insertions(+), 97 deletions(-)
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 6a97c85..0a7416e 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -2786,18 +2786,6 @@
</property>
<property>
- <name>ozone.om.metadata.layout</name>
- <tag>OZONE, OM</tag>
- <value>SIMPLE</value>
- <description>
- This property is used to define the metadata layout of file system
- paths. If it is configured as PREFIX in combination with
- ozone.om.enable.filesystem.paths to true then this allows to perform
- atomic rename and delete of any directory at any level in the namespace.
- Defaulting to SIMPLE. Supported values: SIMPLE and PREFIX.
- </description>
- </property>
- <property>
<name>ozone.directory.deleting.service.interval</name>
<value>1m</value>
<tag>OZONE, PERFORMANCE, OM</tag>
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index c42d640..2dd5113 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -263,12 +263,6 @@ public final class OMConfigKeys {
// atomic rename and delete of any directory at any level in the namespace.
// Defaulting to SIMPLE. Supported values: SIMPLE and PREFIX.
- public static final String OZONE_OM_METADATA_LAYOUT =
- "ozone.om.metadata.layout";
- public static final String OZONE_OM_METADATA_LAYOUT_DEFAULT = "SIMPLE";
-
- public static final String OZONE_OM_METADATA_LAYOUT_PREFIX = "PREFIX";
-
// Default bucket layout used by Ozone Manager during bucket creation
// when a client does not specify the bucket layout option.
public static final String OZONE_DEFAULT_BUCKET_LAYOUT =
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
index a7318a0..56326b4 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
@@ -89,8 +89,6 @@ class OzoneContract extends AbstractFSContract {
if (fsOptimizedServer){
conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS,
true);
- conf.set(OMConfigKeys.OZONE_OM_METADATA_LAYOUT,
- OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX);
}
conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
BucketLayout.LEGACY.name());
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStandardOutputUtil.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStandardOutputUtil.java
new file mode 100644
index 0000000..ec80a49
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStandardOutputUtil.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone;
+
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Utility class to check standard output.
+ */
+public class TestStandardOutputUtil {
+ private final ByteArrayOutputStream outContent =
+ new ByteArrayOutputStream();
+ private final ByteArrayOutputStream errContent =
+ new ByteArrayOutputStream();
+ private final PrintStream originalOut = System.out;
+ private final PrintStream originalErr = System.err;
+ private static final String DEFAULT_ENCODING = StandardCharsets.UTF_8.name();
+
+ /**
+ * Set up fresh output and error streams before test.
+ *
+ * @throws UnsupportedEncodingException
+ */
+ @Before
+ public void setUpStreams() throws UnsupportedEncodingException {
+ System.setOut(new PrintStream(outContent, false, DEFAULT_ENCODING));
+ System.setErr(new PrintStream(errContent, false, DEFAULT_ENCODING));
+ }
+
+ /**
+ * Restore original error and output streams after test.
+ */
+ @After
+ public void restoreStreams() {
+ System.setOut(originalOut);
+ System.setErr(originalErr);
+ }
+
+ public String getOutContentString()
+ throws UnsupportedEncodingException {
+ return getOutContentString(DEFAULT_ENCODING);
+ }
+
+ public String getErrContentString()
+ throws UnsupportedEncodingException {
+ return getErrContentString(DEFAULT_ENCODING);
+ }
+
+ public String getOutContentString(String encoding)
+ throws UnsupportedEncodingException {
+ return outContent.toString(encoding);
+ }
+
+ public String getErrContentString(String encoding)
+ throws UnsupportedEncodingException {
+ return errContent.toString(encoding);
+ }
+
+ public String getDefaultEncoding() {
+ return DEFAULT_ENCODING;
+ }
+
+}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java
index c24ac6b..d772a3f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java
@@ -34,7 +34,6 @@ import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
@@ -108,8 +107,7 @@ public class TestOzoneClientMultipartUploadWithFSO {
@BeforeClass
public static void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
- TestOMRequestUtils.configureFSOptimizedPaths(conf,
- true, OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX);
+ TestOMRequestUtils.configureFSOptimizedPaths(conf, true);
startCluster(conf);
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java
index f94e47e..2e369b4 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
@@ -94,17 +95,18 @@ public class TestReadRetries {
storageContainerLocationClient;
private static final String SCM_ID = UUID.randomUUID().toString();
- private String layoutVersion;
+ private String bucketLayout;
- public TestReadRetries(String layoutVersion) {
- this.layoutVersion = layoutVersion;
+ public TestReadRetries(String bucketLayout) {
+ this.bucketLayout = bucketLayout;
}
@Parameterized.Parameters
public static Collection<Object[]> data() {
return Arrays.asList(
- new Object[]{OMConfigKeys.OZONE_OM_METADATA_LAYOUT_DEFAULT },
- new Object[]{OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX });
+ new Object[]{OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT_DEFAULT},
+ new Object[]{OMConfigKeys.
+ OZONE_BUCKET_LAYOUT_FILE_SYSTEM_OPTIMIZED});
}
/**
@@ -116,7 +118,7 @@ public class TestReadRetries {
OzoneConfiguration conf = new OzoneConfiguration();
conf.setInt(ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, 1);
TestOMRequestUtils.configureFSOptimizedPaths(conf,
- true, layoutVersion);
+ true, BucketLayout.fromString(bucketLayout));
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(3)
.setScmId(SCM_ID)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGeneratorWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGeneratorWithFSO.java
index c776cef..497cdc1 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGeneratorWithFSO.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGeneratorWithFSO.java
@@ -17,7 +17,6 @@
package org.apache.hadoop.ozone.freon;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.request.TestOMRequestUtils;
/**
@@ -28,8 +27,7 @@ public class TestHadoopDirTreeGeneratorWithFSO
protected OzoneConfiguration getOzoneConfiguration() {
OzoneConfiguration conf = new OzoneConfiguration();
- TestOMRequestUtils.configureFSOptimizedPaths(conf,
- true, OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX);
+ TestOMRequestUtils.configureFSOptimizedPaths(conf, true);
return conf;
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestRecursiveAclWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestRecursiveAclWithFSO.java
index bcbea56..b432826 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestRecursiveAclWithFSO.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestRecursiveAclWithFSO.java
@@ -255,8 +255,7 @@ public class TestRecursiveAclWithFSO {
// Note: OM doesn't support live config reloading
conf.setBoolean(OZONE_ACL_ENABLED, true);
- TestOMRequestUtils.configureFSOptimizedPaths(conf, true,
- OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX);
+ TestOMRequestUtils.configureFSOptimizedPaths(conf, true);
cluster =
MiniOzoneCluster.newBuilder(conf).setClusterId(clusterId)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java
index 3b865f6..c84f6b9 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java
@@ -21,34 +21,53 @@ package org.apache.hadoop.ozone.shell;
import org.apache.hadoop.hdds.cli.OzoneAdmin;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.TestStandardOutputUtil;
+import org.apache.hadoop.ozone.client.BucketArgs;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.request.TestOMRequestUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.junit.Assert;
+
+import java.io.UnsupportedEncodingException;
+import java.util.UUID;
import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_ADDRESS_KEY;
/**
* Test for Namespace CLI.
*/
-public class TestNSSummaryAdmin {
+public class TestNSSummaryAdmin extends TestStandardOutputUtil {
+ private static ObjectStore store;
private static OzoneAdmin ozoneAdmin;
private static OzoneConfiguration conf;
private static MiniOzoneCluster cluster;
+ private static String volumeName;
+ private static String bucketOBS;
+ private static String bucketFSO;
+
@BeforeClass
public static void init() throws Exception {
conf = new OzoneConfiguration();
- TestOMRequestUtils.configureFSOptimizedPaths(conf, true,
- OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX);
+ TestOMRequestUtils.configureFSOptimizedPaths(conf, true);
conf.set(OZONE_RECON_ADDRESS_KEY, "localhost:9888");
cluster = MiniOzoneCluster.newBuilder(conf)
.withoutDatanodes().includeRecon(true).build();
cluster.waitForClusterToBeReady();
+ store = cluster.getClient().getObjectStore();
+
// Client uses server conf for this test
ozoneAdmin = new OzoneAdmin(conf);
+
+ volumeName = UUID.randomUUID().toString();
+ bucketOBS = UUID.randomUUID().toString();
+ bucketFSO = UUID.randomUUID().toString();
+ createVolumeAndBuckets();
}
@AfterClass
@@ -58,13 +77,94 @@ public class TestNSSummaryAdmin {
}
}
+ /**
+ * Create OBS and FSO buckets for the tests.
+ * @throws Exception
+ */
+ private static void createVolumeAndBuckets()
+ throws Exception {
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+
+ // Create OBS bucket.
+ BucketArgs bucketArgsOBS = BucketArgs.newBuilder()
+ .setBucketLayout(BucketLayout.OBJECT_STORE)
+ .build();
+ volume.createBucket(bucketOBS, bucketArgsOBS);
+
+ // Create FSO bucket.
+ BucketArgs bucketArgsFSO = BucketArgs.newBuilder()
+ .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED)
+ .build();
+ volume.createBucket(bucketFSO, bucketArgsFSO);
+ }
+
+ /**
+ * Test NSSummaryCLI on root path.
+ */
@Test(timeout = 60000)
- public void testNSSummaryCLI() {
- String[] summaryArgs = {"namespace", "summary", "/"};
- String[] duArgs = {"namespace", "du", "/"};
- String[] duArgsWithOps = {"namespace", "du", "-rfn", "--length=100", "/"};
- String[] quotaArgs = {"namespace", "quota", "/"};
- String[] distArgs = {"namespace", "dist", "/"};
+ public void testNSSummaryCLIRoot() throws UnsupportedEncodingException {
+ // Running on root path.
+ String path = "/";
+ executeAdminCommands(path);
+ // Should throw warning - only buckets can have bucket layout.
+ Assert.assertTrue(
+ getOutContentString().contains(
+ "[Warning] Namespace CLI is only designed for FSO mode."));
+ Assert.assertTrue(getOutContentString()
+ .contains("Put more files into it to visualize DU"));
+ Assert.assertTrue(getOutContentString().contains(
+ "Put more files into it to visualize file size distribution"));
+ }
+
+ /**
+ * Test NSSummaryCLI on FILE_SYSTEM_OPTIMIZED bucket.
+ */
+ @Test(timeout = 60000)
+ public void testNSSummaryCLIFSO() throws UnsupportedEncodingException {
+ // Running on FSO Bucket.
+ String path = "/" + volumeName + "/" + bucketFSO;
+ executeAdminCommands(path);
+ // Should not throw warning, since bucket is in FSO bucket layout.
+ Assert.assertFalse(
+ getOutContentString().contains(
+ "[Warning] Namespace CLI is only designed for FSO mode."));
+ Assert.assertTrue(getOutContentString()
+ .contains("Put more files into it to visualize DU"));
+ Assert.assertTrue(getOutContentString().contains(
+ "Put more files into it to visualize file size distribution"));
+ }
+
+ /**
+ * Test NSSummaryCLI on OBJECT_STORE bucket.
+ */
+ @Test(timeout = 60000)
+ public void testNSSummaryCLIOBS() throws UnsupportedEncodingException {
+ // Running on OBS Bucket.
+ String path = "/" + volumeName + "/" + bucketOBS;
+ executeAdminCommands(path);
+ // Should throw warning, since bucket is in OBS bucket layout.
+ Assert.assertTrue(
+ getOutContentString().contains(
+ "[Warning] Namespace CLI is only designed for FSO mode."));
+ Assert.assertTrue(getOutContentString()
+ .contains("Put more files into it to visualize DU"));
+ Assert.assertTrue(getOutContentString().contains(
+ "Put more files into it to visualize file size distribution"));
+ }
+
+ /**
+ * Execute ozoneAdmin commands on given path.
+ *
+ * @param path
+ */
+ private void executeAdminCommands(String path) {
+ String[] summaryArgs = {"namespace", "summary", path};
+ String[] duArgs = {"namespace", "du", path};
+ String[] duArgsWithOps =
+ {"namespace", "du", "-rfn", "--length=100", path};
+ String[] quotaArgs = {"namespace", "quota", path};
+ String[] distArgs = {"namespace", "dist", path};
ozoneAdmin.execute(summaryArgs);
ozoneAdmin.execute(duArgs);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 2107754..8077d07 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -229,9 +229,6 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HANDLER_COUNT_KEY
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_AUTH_TYPE;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METADATA_LAYOUT;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METADATA_LAYOUT_DEFAULT;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METRICS_SAVE_INTERVAL;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_USER_MAX_VOLUME;
@@ -1352,8 +1349,6 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
* Start service.
*/
public void start() throws IOException {
- initFSOLayout();
-
if (omState == State.BOOTSTRAPPING) {
if (isBootstrapping) {
// Check that all OM configs have been updated with the new OM info.
@@ -1451,7 +1446,6 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
* Restarts the service. This method re-initializes the rpc server.
*/
public void restart() throws IOException {
- initFSOLayout();
setInstanceVariablesFromConf();
LOG.info(buildRpcServerStartMessage("OzoneManager RPC server",
@@ -3607,11 +3601,6 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT);
}
- public String getOMMetadataLayout() {
- return configuration
- .getTrimmed(OZONE_OM_METADATA_LAYOUT, OZONE_OM_METADATA_LAYOUT_DEFAULT);
- }
-
public String getOMDefaultBucketLayout() {
return this.defaultBucketLayout;
}
@@ -3836,19 +3825,6 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
String.valueOf(lvm.getMetadataLayoutVersion()));
}
- private void initFSOLayout() {
- // TODO: Temporary workaround for OM upgrade path and will be replaced once
- // upgrade HDDS-3698 story reaches consensus. Instead of cluster level
- // configuration, OM needs to check this property on every bucket level.
- String metaLayout = getOMMetadataLayout();
- boolean omMetadataLayoutPrefix = StringUtils.equalsIgnoreCase(metaLayout,
- OZONE_OM_METADATA_LAYOUT_PREFIX);
-
- String status = omMetadataLayoutPrefix ? "enabled" : "disabled";
- LOG.info("Configured {}={} and {} optimized OM FS operations",
- OZONE_OM_METADATA_LAYOUT, metaLayout, status);
- }
-
private BucketLayout getBucketLayout() {
return BucketLayout.DEFAULT;
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java
index 234b422..b3acaaa 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java
@@ -139,7 +139,6 @@ public class OMBucketCreateRequest extends OMClientRequest {
OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
getOmRequest());
- String omLayout = ozoneManager.getOMMetadataLayout();
OmBucketInfo omBucketInfo = null;
if (bucketInfo.getBucketLayout() == null || bucketInfo.getBucketLayout()
.equals(BucketLayoutProto.LEGACY)) {
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java
index cd5421a..bec1587 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java
@@ -551,9 +551,6 @@ public final class TestOMRequestUtils {
metadataList.add(HddsProtos.KeyValue.newBuilder().setKey("key2").setValue(
"value2").build());
metadataList.add(HddsProtos.KeyValue.newBuilder().setKey(
- OMConfigKeys.OZONE_OM_METADATA_LAYOUT).setValue(
- OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX).build());
- metadataList.add(HddsProtos.KeyValue.newBuilder().setKey(
OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS).setValue(
"false").build());
return metadataList;
@@ -1066,15 +1063,18 @@ public final class TestOMRequestUtils {
}
public static void configureFSOptimizedPaths(Configuration conf,
- boolean enableFileSystemPaths, String version) {
+ boolean enableFileSystemPaths) {
+ configureFSOptimizedPaths(conf, enableFileSystemPaths,
+ BucketLayout.FILE_SYSTEM_OPTIMIZED);
+ }
+
+ public static void configureFSOptimizedPaths(Configuration conf,
+ boolean enableFileSystemPaths,
+ BucketLayout bucketLayout) {
conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS,
enableFileSystemPaths);
- conf.set(OMConfigKeys.OZONE_OM_METADATA_LAYOUT, version);
- if (StringUtils.equalsIgnoreCase(
- OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX, version)) {
- conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
- BucketLayout.FILE_SYSTEM_OPTIMIZED.name());
- }
+ conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
+ bucketLayout.name());
}
private static BucketLayout getDefaultBucketLayout() {
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestBucketRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestBucketRequest.java
index c066c23..c4b1923 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestBucketRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestBucketRequest.java
@@ -79,7 +79,6 @@ public class TestBucketRequest {
auditLogger = Mockito.mock(AuditLogger.class);
when(ozoneManager.getAuditLogger()).thenReturn(auditLogger);
Mockito.doNothing().when(auditLogger).logWrite(any(AuditMessage.class));
- when(ozoneManager.getOMMetadataLayout()).thenReturn(null);
}
@After
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java
index a460dc8..28ba8de 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.ozone.om.request.bucket;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.request.TestOMRequestUtils;
@@ -41,8 +40,6 @@ public class TestOMBucketCreateRequestWithFSO
@Test
public void testValidateAndUpdateCacheWithFSO() throws Exception {
- when(ozoneManager.getOMMetadataLayout()).thenReturn(
- OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX);
when(ozoneManager.getOMDefaultBucketLayout()).thenReturn(
BucketLayout.FILE_SYSTEM_OPTIMIZED.name());
String volumeName = UUID.randomUUID().toString();
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestWithFSO.java
index ba8d835..36a4c91 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestWithFSO.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestWithFSO.java
@@ -90,8 +90,7 @@ public class TestOMDirectoryCreateRequestWithFSO {
OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS,
folder.newFolder().getAbsolutePath());
- TestOMRequestUtils.configureFSOptimizedPaths(ozoneConfiguration,
- true, OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX);
+ TestOMRequestUtils.configureFSOptimizedPaths(ozoneConfiguration, true);
omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration);
when(ozoneManager.getMetrics()).thenReturn(omMetrics);
when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager);
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/DiskUsageSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/DiskUsageSubCommand.java
index 6a85ba5..3a4a16d 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/DiskUsageSubCommand.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/DiskUsageSubCommand.java
@@ -106,7 +106,7 @@ public class DiskUsageSubCommand implements Callable {
if (duResponse.get("status").equals("PATH_NOT_FOUND")) {
printPathNotFound();
} else {
- if (!parent.isFSOEnabled()) {
+ if (!parent.isFileSystemOptimizedBucket(path)) {
printFSOReminder();
}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/FileSizeDistSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/FileSizeDistSubCommand.java
index 4411405..9f02121 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/FileSizeDistSubCommand.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/FileSizeDistSubCommand.java
@@ -80,7 +80,7 @@ public class FileSizeDistSubCommand implements Callable {
} else if (distResponse.get("status").equals("TYPE_NOT_APPLICABLE")) {
printTypeNA("File Size Distribution");
} else {
- if (!parent.isFSOEnabled()) {
+ if (!parent.isFileSystemOptimizedBucket(path)) {
printFSOReminder();
}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryAdmin.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryAdmin.java
index b89973e..2203658 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryAdmin.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryAdmin.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.ozone.admin.nssummary;
+import org.apache.hadoop.fs.ozone.OzoneClientUtils;
import org.apache.hadoop.hdds.cli.GenericCli;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.cli.OzoneAdmin;
@@ -24,9 +25,18 @@ import org.apache.hadoop.hdds.cli.SubcommandWithParent;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.server.http.HttpConfig;
+import org.apache.hadoop.ozone.OFSPath;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.kohsuke.MetaInfServices;
import picocli.CommandLine;
+import java.io.IOException;
+import java.util.HashSet;
+
import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_ADDRESS_DEFAULT;
import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_ADDRESS_KEY;
import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_HTTPS_ADDRESS_DEFAULT;
@@ -74,10 +84,28 @@ public class NSSummaryAdmin extends GenericCli implements SubcommandWithParent {
return OzoneAdmin.class;
}
- public boolean isFSOEnabled() {
- OzoneConfiguration conf = parent.getOzoneConf();
- return conf.getBoolean("ozone.om.enable.filesystem.paths", false)
- && conf.get("ozone.om.metadata.layout").equalsIgnoreCase("PREFIX");
+ public boolean isFileSystemOptimizedBucket(String path) throws IOException {
+ OFSPath ofsPath = new OFSPath(path);
+
+ OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(getOzoneConfig());
+ ObjectStore objectStore = ozoneClient.getObjectStore();
+
+ try {
+ OzoneBucket bucket = objectStore.getVolume(ofsPath.getVolumeName())
+ .getBucket(ofsPath.getBucketName());
+
+ // Resolve the bucket layout in case this is a Link Bucket.
+ BucketLayout resolvedBucketLayout =
+ OzoneClientUtils.resolveLinkBucketLayout(bucket, objectStore,
+ new HashSet<>());
+
+ return resolvedBucketLayout.isFileSystemOptimized();
+ } catch (IOException e) {
+ System.out.println(
+ "Bucket layout couldn't be verified for path: " + ofsPath +
+ ". Exception: " + e);
+ return false;
+ }
}
/**
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryCLIUtils.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryCLIUtils.java
index dc7d59b..9c56924 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryCLIUtils.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryCLIUtils.java
@@ -154,10 +154,10 @@ public final class NSSummaryCLIUtils {
public static void printFSOReminder() {
printNewLines(1);
- System.out.println("[Warning] FSO is NOT enabled. " +
- "Namespace CLI is only designed for FSO mode.\n" +
- "To enable FSO set ozone.om.enable.filesystem.paths to true " +
- "and ozone.om.metadata.layout to PREFIX.");
+ System.out.println(
+ "[Warning] Namespace CLI is only designed for FSO mode.\n" +
+ "Bucket being accessed must be of type FILE_SYSTEM_OPTIMIZED" +
+ " bucket layout.");
printNewLines(1);
}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/QuotaUsageSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/QuotaUsageSubCommand.java
index 2720411..88a7b2a 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/QuotaUsageSubCommand.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/QuotaUsageSubCommand.java
@@ -80,7 +80,7 @@ public class QuotaUsageSubCommand implements Callable {
} else if (quotaResponse.get("status").equals("TYPE_NOT_APPLICABLE")) {
printTypeNA("Quota");
} else {
- if (!parent.isFSOEnabled()) {
+ if (!parent.isFileSystemOptimizedBucket(path)) {
printFSOReminder();
}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/SummarySubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/SummarySubCommand.java
index ce4616a..c0d2ed7 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/SummarySubCommand.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/SummarySubCommand.java
@@ -76,7 +76,7 @@ public class SummarySubCommand implements Callable<Void> {
if (summaryResponse.get("status").equals("PATH_NOT_FOUND")) {
printPathNotFound();
} else {
- if (!parent.isFSOEnabled()) {
+ if (!parent.isFileSystemOptimizedBucket(path)) {
printFSOReminder();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org