You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ra...@apache.org on 2021/11/14 06:05:54 UTC
[ozone] branch master updated: HDDS-5981. Refactor usage of bucket type. (#2836)
This is an automated email from the ASF dual-hosted git repository.
rakeshr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 71f47d4 HDDS-5981. Refactor usage of bucket type. (#2836)
71f47d4 is described below
commit 71f47d456a4d948ae0afab6870f63bcc2d4d7c26
Author: Mukul Kumar Singh <ms...@apache.org>
AuthorDate: Sun Nov 14 11:35:28 2021 +0530
HDDS-5981. Refactor usage of bucket type. (#2836)
---
.../hadoop/ozone/om/helpers/BucketLayout.java | 17 ++++++++++++
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 7 -----
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 5 ++--
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 4 +--
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 5 ++--
.../om/ratis/utils/OzoneManagerRatisUtils.java | 31 ++++++++++------------
.../hadoop/ozone/om/request/OMClientRequest.java | 22 +++------------
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 3 +--
.../fs/ozone/BasicRootedOzoneFileSystem.java | 4 +--
9 files changed, 44 insertions(+), 54 deletions(-)
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketLayout.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketLayout.java
index 7705186..00118cc 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketLayout.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketLayout.java
@@ -61,6 +61,23 @@ public enum BucketLayout {
}
}
+ public boolean isFileSystemOptimized() {
+ return this.equals(FILE_SYSTEM_OPTIMIZED);
+ }
+
+ public boolean shouldNormalizePaths(boolean enableFileSystemPaths) {
+ switch (this) {
+ case OBJECT_STORE:
+ return false;
+ case FILE_SYSTEM_OPTIMIZED:
+ return true;
+ case LEGACY:
+ return enableFileSystemPaths;
+ default:
+ throw new IllegalArgumentException("Invalid Bucket Layout:" + this);
+ }
+ }
+
public static BucketLayout fromString(String value) {
// This will never be null in production but can be null in mocked
// unit test cases.
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
index 58d7949..130a4eb 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
@@ -206,13 +206,6 @@ public final class OzoneFSUtils {
return keyPath.getNameCount();
}
-
-
-
- public static boolean isFSOptimizedBucket(BucketLayout bucketLayout) {
- return bucketLayout.equals(BucketLayout.FILE_SYSTEM_OPTIMIZED);
- }
-
public static String removeTrailingSlashIfNeeded(String key) {
if (key.endsWith(OZONE_URI_DELIMITER)) {
java.nio.file.Path keyPath = Paths.get(key);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index e0cabad..29ef0b4 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -662,7 +662,7 @@ public class KeyManagerImpl implements KeyManager {
OmKeyInfo value = null;
try {
- if (bucketLayout.equals(BucketLayout.FILE_SYSTEM_OPTIMIZED)) {
+ if (bucketLayout.isFileSystemOptimized()) {
value = getOmKeyInfoFSO(volumeName, bucketName, keyName);
} else {
value = getOmKeyInfo(volumeName, bucketName, keyName);
@@ -3274,8 +3274,7 @@ public class KeyManagerImpl implements KeyManager {
OmBucketInfo buckInfo =
ozoneManager.getMetadataManager().getBucketTable().get(buckKey);
if (buckInfo != null) {
- return buckInfo.getBucketLayout()
- .equals(BucketLayout.FILE_SYSTEM_OPTIMIZED);
+ return buckInfo.getBucketLayout().isFileSystemOptimized();
}
return false;
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index 28a9139..d1095ad 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -279,7 +279,7 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
@Override
public Table<String, OmKeyInfo> getKeyTable(BucketLayout bucketLayout) {
- if (bucketLayout.equals(BucketLayout.FILE_SYSTEM_OPTIMIZED)) {
+ if (bucketLayout.isFileSystemOptimized()) {
return fileTable;
}
return keyTable;
@@ -302,7 +302,7 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
@Override
public Table getOpenKeyTable(BucketLayout bucketLayout) {
- if (bucketLayout.equals(BucketLayout.FILE_SYSTEM_OPTIMIZED)) {
+ if (bucketLayout.isFileSystemOptimized()) {
return openFileTable;
}
return openKeyTable;
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java
index 3c1ce3d..92b7691 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java
@@ -35,7 +35,6 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
-import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.helpers.OMRatisHelper;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils;
import org.apache.hadoop.ozone.om.request.OMClientRequest;
@@ -157,7 +156,7 @@ public class TrashOzoneFileSystem extends FileSystem {
OFSPath dstPath = new OFSPath(dst);
OmBucketInfo bucket = ozoneManager.getBucketInfo(srcPath.getVolumeName(),
srcPath.getBucketName());
- if (OzoneFSUtils.isFSOptimizedBucket(bucket.getBucketLayout())) {
+ if (bucket.getBucketLayout().isFileSystemOptimized()) {
return renameFSO(srcPath, dstPath);
}
Preconditions.checkArgument(srcPath.getBucketName().
@@ -191,7 +190,7 @@ public class TrashOzoneFileSystem extends FileSystem {
OFSPath srcPath = new OFSPath(path);
OmBucketInfo bucket = ozoneManager.getBucketInfo(srcPath.getVolumeName(),
srcPath.getBucketName());
- if (OzoneFSUtils.isFSOptimizedBucket(bucket.getBucketLayout())) {
+ if (bucket.getBucketLayout().isFileSystemOptimized()) {
return deleteFSO(srcPath);
}
DeleteIterator iterator = new DeleteIterator(path, true);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
index b4d2e29..6fc029d 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
@@ -185,7 +185,7 @@ public final class OzoneManagerRatisUtils {
keyArgs = omRequest.getAllocateBlockRequest().getKeyArgs();
bucketLayout = getBucketLayout(keyArgs.getVolumeName(),
keyArgs.getBucketName(), ozoneManager);
- if (bucketLayout.equals(BucketLayout.FILE_SYSTEM_OPTIMIZED)) {
+ if (bucketLayout.isFileSystemOptimized()) {
return new OMAllocateBlockRequestWithFSO(omRequest, bucketLayout);
}
return new OMAllocateBlockRequest(omRequest, bucketLayout);
@@ -193,7 +193,7 @@ public final class OzoneManagerRatisUtils {
keyArgs = omRequest.getCreateKeyRequest().getKeyArgs();
bucketLayout = getBucketLayout(keyArgs.getVolumeName(),
keyArgs.getBucketName(), ozoneManager);
- if (bucketLayout.equals(BucketLayout.FILE_SYSTEM_OPTIMIZED)) {
+ if (bucketLayout.isFileSystemOptimized()) {
return new OMKeyCreateRequestWithFSO(omRequest, bucketLayout);
}
return new OMKeyCreateRequest(omRequest, bucketLayout);
@@ -201,7 +201,7 @@ public final class OzoneManagerRatisUtils {
keyArgs = omRequest.getCommitKeyRequest().getKeyArgs();
bucketLayout = getBucketLayout(keyArgs.getVolumeName(),
keyArgs.getBucketName(), ozoneManager);
- if (bucketLayout.equals(BucketLayout.FILE_SYSTEM_OPTIMIZED)) {
+ if (bucketLayout.isFileSystemOptimized()) {
return new OMKeyCommitRequestWithFSO(omRequest, bucketLayout);
}
return new OMKeyCommitRequest(omRequest, bucketLayout);
@@ -209,7 +209,7 @@ public final class OzoneManagerRatisUtils {
keyArgs = omRequest.getDeleteKeyRequest().getKeyArgs();
bucketLayout = getBucketLayout(keyArgs.getVolumeName(),
keyArgs.getBucketName(), ozoneManager);
- if (bucketLayout.equals(BucketLayout.FILE_SYSTEM_OPTIMIZED)) {
+ if (bucketLayout.isFileSystemOptimized()) {
return new OMKeyDeleteRequestWithFSO(omRequest, bucketLayout);
}
return new OMKeyDeleteRequest(omRequest, bucketLayout);
@@ -219,7 +219,7 @@ public final class OzoneManagerRatisUtils {
keyArgs = omRequest.getRenameKeyRequest().getKeyArgs();
bucketLayout = getBucketLayout(keyArgs.getVolumeName(),
keyArgs.getBucketName(), ozoneManager);
- if (bucketLayout.equals(BucketLayout.FILE_SYSTEM_OPTIMIZED)) {
+ if (bucketLayout.isFileSystemOptimized()) {
return new OMKeyRenameRequestWithFSO(omRequest, bucketLayout);
}
return new OMKeyRenameRequest(omRequest, bucketLayout);
@@ -229,7 +229,7 @@ public final class OzoneManagerRatisUtils {
keyArgs = omRequest.getCreateDirectoryRequest().getKeyArgs();
bucketLayout = getBucketLayout(keyArgs.getVolumeName(),
keyArgs.getBucketName(), ozoneManager);
- if (bucketLayout.equals(BucketLayout.FILE_SYSTEM_OPTIMIZED)) {
+ if (bucketLayout.isFileSystemOptimized()) {
return new OMDirectoryCreateRequestWithFSO(omRequest, bucketLayout);
}
return new OMDirectoryCreateRequest(omRequest, bucketLayout);
@@ -237,7 +237,7 @@ public final class OzoneManagerRatisUtils {
keyArgs = omRequest.getCreateFileRequest().getKeyArgs();
bucketLayout = getBucketLayout(keyArgs.getVolumeName(),
keyArgs.getBucketName(), ozoneManager);
- if (bucketLayout.equals(BucketLayout.FILE_SYSTEM_OPTIMIZED)) {
+ if (bucketLayout.isFileSystemOptimized()) {
return new OMFileCreateRequestWithFSO(omRequest, bucketLayout);
}
return new OMFileCreateRequest(omRequest);
@@ -250,7 +250,7 @@ public final class OzoneManagerRatisUtils {
keyArgs = omRequest.getInitiateMultiPartUploadRequest().getKeyArgs();
bucketLayout = getBucketLayout(keyArgs.getVolumeName(),
keyArgs.getBucketName(), ozoneManager);
- if (bucketLayout.equals(BucketLayout.FILE_SYSTEM_OPTIMIZED)) {
+ if (bucketLayout.isFileSystemOptimized()) {
return new S3InitiateMultipartUploadRequestWithFSO(omRequest,
bucketLayout);
}
@@ -259,7 +259,7 @@ public final class OzoneManagerRatisUtils {
keyArgs = omRequest.getCommitMultiPartUploadRequest().getKeyArgs();
bucketLayout = getBucketLayout(keyArgs.getVolumeName(),
keyArgs.getBucketName(), ozoneManager);
- if (bucketLayout.equals(BucketLayout.FILE_SYSTEM_OPTIMIZED)) {
+ if (bucketLayout.isFileSystemOptimized()) {
return new S3MultipartUploadCommitPartRequestWithFSO(omRequest,
bucketLayout);
}
@@ -268,7 +268,7 @@ public final class OzoneManagerRatisUtils {
keyArgs = omRequest.getAbortMultiPartUploadRequest().getKeyArgs();
bucketLayout = getBucketLayout(keyArgs.getVolumeName(),
keyArgs.getBucketName(), ozoneManager);
- if (bucketLayout.equals(BucketLayout.FILE_SYSTEM_OPTIMIZED)) {
+ if (bucketLayout.isFileSystemOptimized()) {
return new S3MultipartUploadAbortRequestWithFSO(omRequest,
bucketLayout);
}
@@ -277,7 +277,7 @@ public final class OzoneManagerRatisUtils {
keyArgs = omRequest.getCompleteMultiPartUploadRequest().getKeyArgs();
bucketLayout = getBucketLayout(keyArgs.getVolumeName(),
keyArgs.getBucketName(), ozoneManager);
- if (bucketLayout.equals(BucketLayout.FILE_SYSTEM_OPTIMIZED)) {
+ if (bucketLayout.isFileSystemOptimized()) {
return new S3MultipartUploadCompleteRequestWithFSO(omRequest,
bucketLayout);
}
@@ -322,8 +322,7 @@ public final class OzoneManagerRatisUtils {
} else if (ObjectType.KEY == type) {
OMKeyAddAclRequest aclReq = new OMKeyAddAclRequest(omRequest);
BucketLayout bucketLayout = aclReq.getBucketLayout(ozoneManager);
- if (BucketLayout.FILE_SYSTEM_OPTIMIZED
- .equals(bucketLayout)) {
+ if (bucketLayout.isFileSystemOptimized()) {
return new OMKeyAddAclRequestWithFSO(omRequest);
}
return aclReq;
@@ -339,8 +338,7 @@ public final class OzoneManagerRatisUtils {
} else if (ObjectType.KEY == type) {
OMKeyRemoveAclRequest aclReq = new OMKeyRemoveAclRequest(omRequest);
BucketLayout bucketLayout = aclReq.getBucketLayout(ozoneManager);
- if (BucketLayout.FILE_SYSTEM_OPTIMIZED
- .equals(bucketLayout)) {
+ if (bucketLayout.isFileSystemOptimized()) {
return new OMKeyRemoveAclRequestWithFSO(omRequest);
}
return aclReq;
@@ -356,8 +354,7 @@ public final class OzoneManagerRatisUtils {
} else if (ObjectType.KEY == type) {
OMKeySetAclRequest aclReq = new OMKeySetAclRequest(omRequest);
BucketLayout bucketLayout = aclReq.getBucketLayout(ozoneManager);
- if (BucketLayout.FILE_SYSTEM_OPTIMIZED
- .equals(bucketLayout)) {
+ if (bucketLayout.isFileSystemOptimized()) {
return new OMKeySetAclRequestWithFSO(omRequest);
}
return aclReq;
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java
index 6cb2fa2..fd1e01e 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java
@@ -411,27 +411,13 @@ public abstract class OMClientRequest implements RequestAuditor {
public static String validateAndNormalizeKey(boolean enableFileSystemPaths,
String keyPath, BucketLayout bucketLayout) throws OMException {
LOG.debug("Bucket Layout: {}", bucketLayout);
- if (bucketLayout.equals(BucketLayout.OBJECT_STORE)) {
- // If bucket layout is OBJECT_STORE than we don't
- // need to normalize the key.
- return keyPath;
- } else if (bucketLayout.equals(BucketLayout.FILE_SYSTEM_OPTIMIZED)) {
+ if (bucketLayout.shouldNormalizePaths(enableFileSystemPaths)) {
keyPath = validateAndNormalizeKey(true, keyPath);
if (keyPath.endsWith("/")) {
throw new OMException(
- "Invalid KeyPath, key names with trailing / " + "are not allowed."
- + keyPath, OMException.ResultCodes.INVALID_KEY_NAME);
- }
- } else {
- // In this case our bucket layout is LEGACY, we will normalize
- // the key if 'enableFileSystemPaths' flag is true.
- keyPath = validateAndNormalizeKey(enableFileSystemPaths, keyPath);
- if (enableFileSystemPaths) {
- if (keyPath.endsWith("/")) {
- throw new OMException(
- "Invalid KeyPath, key names with trailing / " + "are not allowed."
- + keyPath, OMException.ResultCodes.INVALID_KEY_NAME);
- }
+ "Invalid KeyPath, key names with trailing / "
+ + "are not allowed." + keyPath,
+ OMException.ResultCodes.INVALID_KEY_NAME);
}
}
return keyPath;
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
index 6bcb689..b966825 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
@@ -53,7 +53,6 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
-import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenRenewer;
@@ -545,6 +544,6 @@ public class BasicOzoneClientAdapterImpl implements OzoneClientAdapter {
@Override
public boolean isFSOptimizedBucket() {
- return bucket.getBucketLayout().equals(BucketLayout.FILE_SYSTEM_OPTIMIZED);
+ return bucket.getBucketLayout().isFileSystemOptimized();
}
}
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
index 144e9e5..35065f0 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
@@ -310,7 +310,7 @@ public class BasicRootedOzoneFileSystem extends FileSystem {
throw new IOException("Cannot rename a key to a different bucket");
}
OzoneBucket bucket = adapterImpl.getBucket(ofsSrc, false);
- if (OzoneFSUtils.isFSOptimizedBucket(bucket.getBucketLayout())) {
+ if (bucket.getBucketLayout().isFileSystemOptimized()) {
return renameFSO(bucket, ofsSrc, ofsDst);
}
@@ -532,7 +532,7 @@ public class BasicRootedOzoneFileSystem extends FileSystem {
if (!ofsPath.isVolume() && !ofsPath.isBucket()) {
OzoneBucket bucket = adapterImpl.getBucket(ofsPath, false);
- if (OzoneFSUtils.isFSOptimizedBucket(bucket.getBucketLayout())) {
+ if (bucket.getBucketLayout().isFileSystemOptimized()) {
String ofsKeyPath = ofsPath.getNonKeyPathNoPrefixDelim() +
OZONE_URI_DELIMITER + ofsPath.getKeyName();
return adapterImpl.deleteObject(ofsKeyPath, recursive);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org