You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by jy...@apache.org on 2022/05/26 07:48:02 UTC
[ozone] branch master updated: HDDS-6681. Post-Finalize behaviour for Bucket Layout feature. (#3411)
This is an automated email from the ASF dual-hosted git repository.
jyotinder pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new ddbe71d3f9 HDDS-6681. Post-Finalize behaviour for Bucket Layout feature. (#3411)
ddbe71d3f9 is described below
commit ddbe71d3f90ae14ecf29f51c2265f1f9c3171668
Author: Jyotinder Singh <jy...@gmail.com>
AuthorDate: Thu May 26 13:17:57 2022 +0530
HDDS-6681. Post-Finalize behaviour for Bucket Layout feature. (#3411)
---
.../org/apache/hadoop/ozone/ClientVersion.java | 4 +
.../hadoop/ozone/om/helpers/BucketLayout.java | 17 ++
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 4 +
.../om/request/bucket/OMBucketDeleteRequest.java | 34 +++
.../om/request/file/OMDirectoryCreateRequest.java | 31 +++
.../ozone/om/request/file/OMFileCreateRequest.java | 35 ++-
.../om/request/key/OMAllocateBlockRequest.java | 29 +++
.../ozone/om/request/key/OMKeyCommitRequest.java | 29 +++
.../ozone/om/request/key/OMKeyCreateRequest.java | 29 +++
.../ozone/om/request/key/OMKeyDeleteRequest.java | 35 +++
.../ozone/om/request/key/OMKeyRenameRequest.java | 35 +++
.../ozone/om/request/key/OMKeysDeleteRequest.java | 35 +++
.../ozone/om/request/key/OMKeysRenameRequest.java | 35 +++
.../S3InitiateMultipartUploadRequest.java | 30 +++
.../multipart/S3MultipartUploadAbortRequest.java | 29 +++
.../S3MultipartUploadCommitPartRequest.java | 29 +++
.../S3MultipartUploadCompleteRequest.java | 29 +++
.../om/request/validation/RequestValidations.java | 2 +-
.../protocolPB/OzoneManagerRequestHandler.java | 234 +++++++++++++++++++++
19 files changed, 703 insertions(+), 2 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java
index b8e2542bde..cc6695dc7d 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java
@@ -38,6 +38,10 @@ public enum ClientVersion implements ComponentVersion {
ERASURE_CODING_SUPPORT(2, "This client version has support for Erasure"
+ " Coding."),
+ BUCKET_LAYOUT_SUPPORT(3,
+ "This client version has support for Object Store and File " +
+ "System Optimized Bucket Layouts."),
+
FUTURE_VERSION(-1, "Used internally when the server side is older and an"
+ " unknown client version has arrived from the client.");
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketLayout.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketLayout.java
index a9803e03e4..8a17777b11 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketLayout.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketLayout.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.om.helpers;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
/**
@@ -88,4 +89,20 @@ public enum BucketLayout {
// Added safer `isBlank` check for unit test cases.
return StringUtils.isBlank(value) ? LEGACY : BucketLayout.valueOf(value);
}
+
+ /**
+ * Helper method for upgrade scenarios. Throws an exception if a bucket layout
+ * is not supported on an older client.
+ *
+ * @throws OMException if bucket layout is not supported on older clients.
+ */
+ public void validateSupportedOperation() throws OMException {
+ // Older clients do not support any bucket layout other than LEGACY.
+ if (!isLegacy()) {
+ throw new OMException("Client is attempting to modify a bucket which" +
+ " uses non-LEGACY bucket layout features. Please upgrade the client" +
+ " to a compatible version before performing this operation.",
+ OMException.ResultCodes.NOT_SUPPORTED_OPERATION);
+ }
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java
index b27ae730ae..41017f1d86 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java
@@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.om;
import com.google.common.base.Preconditions;
import com.google.protobuf.RpcController;
+import org.apache.hadoop.ozone.ClientVersion;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -446,6 +447,7 @@ public class TrashOzoneFileSystem extends FileSystem {
try {
omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder()
.setClientId(CLIENT_ID.toString())
+ .setVersion(ClientVersion.CURRENT_VERSION)
.setUserInfo(getUserInfo())
.setRenameKeyRequest(renameKeyRequest)
.setCmdType(OzoneManagerProtocolProtos.Type.RenameKey)
@@ -510,6 +512,7 @@ public class TrashOzoneFileSystem extends FileSystem {
omRequest =
OzoneManagerProtocolProtos.OMRequest.newBuilder()
.setClientId(CLIENT_ID.toString())
+ .setVersion(ClientVersion.CURRENT_VERSION)
.setUserInfo(getUserInfo())
.setDeleteKeyRequest(deleteKeyRequest)
.setCmdType(OzoneManagerProtocolProtos.Type.DeleteKey)
@@ -577,6 +580,7 @@ public class TrashOzoneFileSystem extends FileSystem {
try {
omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder()
.setClientId(CLIENT_ID.toString())
+ .setVersion(ClientVersion.CURRENT_VERSION)
.setUserInfo(getUserInfo())
.setDeleteKeysRequest(deleteKeysRequest)
.setCmdType(OzoneManagerProtocolProtos.Type.DeleteKeys)
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java
index 4e234aa69a..12dfd4d3ca 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java
@@ -22,10 +22,15 @@ import java.io.IOException;
import java.util.Map;
import com.google.common.base.Optional;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
+import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator;
+import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase;
+import org.apache.hadoop.ozone.om.request.validation.ValidationCondition;
+import org.apache.hadoop.ozone.om.request.validation.ValidationContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,6 +47,7 @@ import org.apache.hadoop.ozone.om.response.bucket.OMBucketDeleteResponse;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.DeleteBucketRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
@@ -192,4 +198,32 @@ public class OMBucketDeleteRequest extends OMClientRequest {
}
}
+ /**
+ * Validates bucket delete requests.
+ * Handles the cases where an older client attempts to delete a bucket
+ * a new bucket layout.
+ * We do not want to allow this to happen, since this would cause the client
+ * to be able to delete buckets it cannot understand.
+ *
+ * @param req - the request to validate
+ * @param ctx - the validation context
+ * @return the validated request
+ * @throws OMException if the request is invalid
+ */
+ @RequestFeatureValidator(
+ conditions = ValidationCondition.OLDER_CLIENT_REQUESTS,
+ processingPhase = RequestProcessingPhase.PRE_PROCESS,
+ requestType = Type.DeleteBucket
+ )
+ public static OMRequest blockBucketDeleteWithBucketLayoutFromOldClient(
+ OMRequest req, ValidationContext ctx) throws IOException {
+ DeleteBucketRequest request = req.getDeleteBucketRequest();
+
+ if (request.hasBucketName() && request.hasVolumeName()) {
+ BucketLayout bucketLayout = ctx.getBucketLayout(
+ request.getVolumeName(), request.getBucketName());
+ bucketLayout.validateSupportedOperation();
+ }
+ return req;
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java
index 62e8bc2076..729cf5a876 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java
@@ -415,4 +415,35 @@ public class OMDirectoryCreateRequest extends OMKeyRequest {
}
return req;
}
+
+ /**
+ * Validates directory create requests.
+ * Handles the cases where an older client attempts to create a directory
+ * inside a bucket with a non LEGACY bucket layout.
+ * We do not want an older client modifying a bucket that it cannot
+ * understand.
+ *
+ * @param req - the request to validate
+ * @param ctx - the validation context
+ * @return the validated request
+ * @throws OMException if the request is invalid
+ */
+ @RequestFeatureValidator(
+ conditions = ValidationCondition.OLDER_CLIENT_REQUESTS,
+ processingPhase = RequestProcessingPhase.PRE_PROCESS,
+ requestType = Type.CreateDirectory
+ )
+ public static OMRequest blockCreateDirectoryWithBucketLayoutFromOldClient(
+ OMRequest req, ValidationContext ctx) throws IOException {
+ if (req.getCreateDirectoryRequest().hasKeyArgs()) {
+ KeyArgs keyArgs = req.getCreateDirectoryRequest().getKeyArgs();
+
+ if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) {
+ BucketLayout bucketLayout = ctx.getBucketLayout(
+ keyArgs.getVolumeName(), keyArgs.getBucketName());
+ bucketLayout.validateSupportedOperation();
+ }
+ }
+ return req;
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
index fdfb4119bd..634ba81f2a 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
@@ -42,6 +42,8 @@ import org.apache.hadoop.ozone.om.request.validation.ValidationCondition;
import org.apache.hadoop.ozone.om.request.validation.ValidationContext;
import org.apache.hadoop.ozone.om.response.file.OMFileCreateResponse;
import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -60,7 +62,6 @@ import org.apache.hadoop.ozone.om.request.key.OMKeyRequest;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateFileRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateFileResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
@@ -408,4 +409,36 @@ public class OMFileCreateRequest extends OMKeyRequest {
}
return req;
}
+
+ /**
+ * Validates file create requests.
+ * Handles the cases where an older client attempts to create a file
+ * inside a bucket with a non LEGACY bucket layout.
+ * We do not want an older client modifying a bucket that it cannot
+ * understand.
+ *
+ * @param req - the request to validate
+ * @param ctx - the validation context
+ * @return the validated request
+ * @throws OMException if the request is invalid
+ */
+ @RequestFeatureValidator(
+ conditions = ValidationCondition.OLDER_CLIENT_REQUESTS,
+ processingPhase = RequestProcessingPhase.PRE_PROCESS,
+ requestType = Type.CreateFile
+ )
+ public static OMRequest blockCreateFileWithBucketLayoutFromOldClient(
+ OMRequest req, ValidationContext ctx) throws IOException {
+ if (req.getCreateFileRequest().hasKeyArgs()) {
+
+ KeyArgs keyArgs = req.getCreateFileRequest().getKeyArgs();
+
+ if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) {
+ BucketLayout bucketLayout = ctx.getBucketLayout(
+ keyArgs.getVolumeName(), keyArgs.getBucketName());
+ bucketLayout.validateSupportedOperation();
+ }
+ }
+ return req;
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
index a2cd19ab19..ecc2bf06c5 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
@@ -284,4 +284,33 @@ public class OMAllocateBlockRequest extends OMKeyRequest {
}
return req;
}
+
+ /**
+ * Validates block allocation requests.
+ * We do not want to allow older clients to create block allocation requests
+ * for keys that are present in buckets which use non LEGACY layouts.
+ *
+ * @param req - the request to validate
+ * @param ctx - the validation context
+ * @return the validated request
+ * @throws OMException if the request is invalid
+ */
+ @RequestFeatureValidator(
+ conditions = ValidationCondition.OLDER_CLIENT_REQUESTS,
+ processingPhase = RequestProcessingPhase.PRE_PROCESS,
+ requestType = Type.AllocateBlock
+ )
+ public static OMRequest blockAllocateBlockWithBucketLayoutFromOldClient(
+ OMRequest req, ValidationContext ctx) throws IOException {
+ if (req.getAllocateBlockRequest().hasKeyArgs()) {
+ KeyArgs keyArgs = req.getAllocateBlockRequest().getKeyArgs();
+
+ if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) {
+ BucketLayout bucketLayout = ctx.getBucketLayout(
+ keyArgs.getVolumeName(), keyArgs.getBucketName());
+ bucketLayout.validateSupportedOperation();
+ }
+ }
+ return req;
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
index 6f9cafe4ec..ab903416e2 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
@@ -340,4 +340,33 @@ public class OMKeyCommitRequest extends OMKeyRequest {
}
return req;
}
+
+ /**
+ * Validates key commit requests.
+ * We do not want to allow older clients to commit keys associated with
+ * buckets which use non LEGACY layouts.
+ *
+ * @param req - the request to validate
+ * @param ctx - the validation context
+ * @return the validated request
+ * @throws OMException if the request is invalid
+ */
+ @RequestFeatureValidator(
+ conditions = ValidationCondition.OLDER_CLIENT_REQUESTS,
+ processingPhase = RequestProcessingPhase.PRE_PROCESS,
+ requestType = Type.CommitKey
+ )
+ public static OMRequest blockCommitKeyWithBucketLayoutFromOldClient(
+ OMRequest req, ValidationContext ctx) throws IOException {
+ if (req.getCommitKeyRequest().hasKeyArgs()) {
+ KeyArgs keyArgs = req.getCommitKeyRequest().getKeyArgs();
+
+ if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) {
+ BucketLayout bucketLayout = ctx.getBucketLayout(
+ keyArgs.getVolumeName(), keyArgs.getBucketName());
+ bucketLayout.validateSupportedOperation();
+ }
+ }
+ return req;
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
index 906ea59754..99986a7410 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
@@ -395,4 +395,33 @@ public class OMKeyCreateRequest extends OMKeyRequest {
}
return req;
}
+
+ /**
+ * Validates key create requests.
+ * We do not want to allow older clients to create keys in buckets which use
+ * non LEGACY layouts.
+ *
+ * @param req - the request to validate
+ * @param ctx - the validation context
+ * @return the validated request
+ * @throws OMException if the request is invalid
+ */
+ @RequestFeatureValidator(
+ conditions = ValidationCondition.OLDER_CLIENT_REQUESTS,
+ processingPhase = RequestProcessingPhase.PRE_PROCESS,
+ requestType = Type.CreateKey
+ )
+ public static OMRequest blockCreateKeyWithBucketLayoutFromOldClient(
+ OMRequest req, ValidationContext ctx) throws IOException {
+ if (req.getCreateKeyRequest().hasKeyArgs()) {
+ KeyArgs keyArgs = req.getCreateKeyRequest().getKeyArgs();
+
+ if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) {
+ BucketLayout bucketLayout = ctx.getBucketLayout(
+ keyArgs.getVolumeName(), keyArgs.getBucketName());
+ bucketLayout.validateSupportedOperation();
+ }
+ }
+ return req;
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
index d54e4ebddd..1398c0b707 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
@@ -26,6 +26,10 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
+import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator;
+import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase;
+import org.apache.hadoop.ozone.om.request.validation.ValidationCondition;
+import org.apache.hadoop.ozone.om.request.validation.ValidationContext;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.slf4j.Logger;
@@ -42,6 +46,8 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.om.response.key.OMKeyDeleteResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
@@ -209,4 +215,33 @@ public class OMKeyDeleteRequest extends OMKeyRequest {
return omClientResponse;
}
+
+ /**
+ * Validates key delete requests.
+ * We do not want to allow older clients to delete keys in buckets which use
+ * non LEGACY layouts.
+ *
+ * @param req - the request to validate
+ * @param ctx - the validation context
+ * @return the validated request
+ * @throws OMException if the request is invalid
+ */
+ @RequestFeatureValidator(
+ conditions = ValidationCondition.OLDER_CLIENT_REQUESTS,
+ processingPhase = RequestProcessingPhase.PRE_PROCESS,
+ requestType = Type.DeleteKey
+ )
+ public static OMRequest blockDeleteKeyWithBucketLayoutFromOldClient(
+ OMRequest req, ValidationContext ctx) throws IOException {
+ if (req.getDeleteKeyRequest().hasKeyArgs()) {
+ KeyArgs keyArgs = req.getDeleteKeyRequest().getKeyArgs();
+
+ if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) {
+ BucketLayout bucketLayout = ctx.getBucketLayout(
+ keyArgs.getVolumeName(), keyArgs.getBucketName());
+ bucketLayout.validateSupportedOperation();
+ }
+ }
+ return req;
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
index 2e25b567c7..033f622782 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
@@ -29,6 +29,10 @@ import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
+import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator;
+import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase;
+import org.apache.hadoop.ozone.om.request.validation.ValidationCondition;
+import org.apache.hadoop.ozone.om.request.validation.ValidationContext;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.slf4j.Logger;
@@ -44,6 +48,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.om.response.key.OMKeyRenameResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.KeyArgs;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
@@ -250,4 +255,34 @@ public class OMKeyRenameRequest extends OMKeyRequest {
auditMap.put(OzoneConsts.DST_KEY, renameKeyRequest.getToKeyName());
return auditMap;
}
+
+
+ /**
+ * Validates rename key requests.
+ * We do not want to allow older clients to rename keys in buckets which use
+ * non LEGACY layouts.
+ *
+ * @param req - the request to validate
+ * @param ctx - the validation context
+ * @return the validated request
+ * @throws OMException if the request is invalid
+ */
+ @RequestFeatureValidator(
+ conditions = ValidationCondition.OLDER_CLIENT_REQUESTS,
+ processingPhase = RequestProcessingPhase.PRE_PROCESS,
+ requestType = Type.RenameKey
+ )
+ public static OMRequest blockRenameKeyWithBucketLayoutFromOldClient(
+ OMRequest req, ValidationContext ctx) throws IOException {
+ if (req.getRenameKeyRequest().hasKeyArgs()) {
+ KeyArgs keyArgs = req.getRenameKeyRequest().getKeyArgs();
+
+ if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) {
+ BucketLayout bucketLayout = ctx.getBucketLayout(
+ keyArgs.getVolumeName(), keyArgs.getBucketName());
+ bucketLayout.validateSupportedOperation();
+ }
+ }
+ return req;
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java
index 9c34242f26..1781612d0b 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java
@@ -27,15 +27,22 @@ import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.ResolvedBucket;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
+import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator;
+import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase;
+import org.apache.hadoop.ozone.om.request.validation.ValidationCondition;
+import org.apache.hadoop.ozone.om.request.validation.ValidationContext;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.om.response.key.OMKeysDeleteResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyArgs;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeysRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeysResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
@@ -295,4 +302,32 @@ public class OMKeysDeleteRequest extends OMKeyRequest {
auditMap.put(UNDELETED_KEYS_LIST, String.join(",", unDeletedKeys));
}
+ /**
+ * Validates delete key requests.
+ * We do not want to allow older clients to delete keys in buckets which use
+ * non LEGACY layouts.
+ *
+ * @param req - the request to validate
+ * @param ctx - the validation context
+ * @return the validated request
+ * @throws OMException if the request is invalid
+ */
+ @RequestFeatureValidator(
+ conditions = ValidationCondition.OLDER_CLIENT_REQUESTS,
+ processingPhase = RequestProcessingPhase.PRE_PROCESS,
+ requestType = Type.DeleteKeys
+ )
+ public static OMRequest blockDeleteKeysWithBucketLayoutFromOldClient(
+ OMRequest req, ValidationContext ctx) throws IOException {
+ if (req.getDeleteKeysRequest().hasDeleteKeys()) {
+ DeleteKeyArgs keyArgs = req.getDeleteKeysRequest().getDeleteKeys();
+
+ if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) {
+ BucketLayout bucketLayout = ctx.getBucketLayout(
+ keyArgs.getVolumeName(), keyArgs.getBucketName());
+ bucketLayout.validateSupportedOperation();
+ }
+ }
+ return req;
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java
index 4427e6d64e..bd6e98cc91 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java
@@ -28,14 +28,20 @@ import org.apache.hadoop.ozone.audit.OMAction;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.ResolvedBucket;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmRenameKeys;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
+import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator;
+import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase;
+import org.apache.hadoop.ozone.om.request.validation.ValidationCondition;
+import org.apache.hadoop.ozone.om.request.validation.ValidationContext;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.om.response.key.OMKeysRenameResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeysArgs;
@@ -277,4 +283,33 @@ public class OMKeysRenameRequest extends OMKeyRequest {
auditMap.put(UNRENAMED_KEYS_MAP, unRenameKeysMap.toString());
return auditMap;
}
+
+ /**
+ * Validates rename keys requests.
+ * We do not want to allow older clients to rename keys in buckets which use
+ * non LEGACY layouts.
+ *
+ * @param req - the request to validate
+ * @param ctx - the validation context
+ * @return the validated request
+ * @throws OMException if the request is invalid
+ */
+ @RequestFeatureValidator(
+ conditions = ValidationCondition.OLDER_CLIENT_REQUESTS,
+ processingPhase = RequestProcessingPhase.PRE_PROCESS,
+ requestType = Type.RenameKeys
+ )
+ public static OMRequest blockRenameKeysWithBucketLayoutFromOldClient(
+ OMRequest req, ValidationContext ctx) throws IOException {
+ if (req.getRenameKeysRequest().hasRenameKeysArgs()) {
+ RenameKeysArgs keyArgs = req.getRenameKeysRequest().getRenameKeysArgs();
+
+ if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) {
+ BucketLayout bucketLayout = ctx.getBucketLayout(
+ keyArgs.getVolumeName(), keyArgs.getBucketName());
+ bucketLayout.validateSupportedOperation();
+ }
+ }
+ return req;
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java
index 4aaec04bf3..9be37ecd52 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java
@@ -298,4 +298,34 @@ public class S3InitiateMultipartUploadRequest extends OMKeyRequest {
}
return req;
}
+
+
+ /**
+ * Validates S3 initiate MPU requests.
+ * We do not want to allow older clients to initiate MPU to buckets which
+ * use non LEGACY layouts.
+ *
+ * @param req - the request to validate
+ * @param ctx - the validation context
+ * @return the validated request
+ * @throws OMException if the request is invalid
+ */
+ @RequestFeatureValidator(
+ conditions = ValidationCondition.OLDER_CLIENT_REQUESTS,
+ processingPhase = RequestProcessingPhase.PRE_PROCESS,
+ requestType = Type.InitiateMultiPartUpload
+ )
+ public static OMRequest blockInitiateMPUWithBucketLayoutFromOldClient(
+ OMRequest req, ValidationContext ctx) throws IOException {
+ if (req.getInitiateMultiPartUploadRequest().hasKeyArgs()) {
+ KeyArgs keyArgs = req.getInitiateMultiPartUploadRequest().getKeyArgs();
+
+ if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) {
+ BucketLayout bucketLayout = ctx.getBucketLayout(
+ keyArgs.getVolumeName(), keyArgs.getBucketName());
+ bucketLayout.validateSupportedOperation();
+ }
+ }
+ return req;
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java
index 0959c71988..abb3a45a33 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java
@@ -282,4 +282,33 @@ public class S3MultipartUploadAbortRequest extends OMKeyRequest {
}
return req;
}
+
+ /**
+ * Validates S3 MPU abort requests.
+ * We do not want to allow older clients to abort MPU operations in
+ * buckets which use non LEGACY layouts.
+ *
+ * @param req - the request to validate
+ * @param ctx - the validation context
+ * @return the validated request
+ * @throws OMException if the request is invalid
+ */
+ @RequestFeatureValidator(
+ conditions = ValidationCondition.OLDER_CLIENT_REQUESTS,
+ processingPhase = RequestProcessingPhase.PRE_PROCESS,
+ requestType = Type.AbortMultiPartUpload
+ )
+ public static OMRequest blockMPUAbortWithBucketLayoutFromOldClient(
+ OMRequest req, ValidationContext ctx) throws IOException {
+ if (req.getAbortMultiPartUploadRequest().hasKeyArgs()) {
+ KeyArgs keyArgs = req.getAbortMultiPartUploadRequest().getKeyArgs();
+
+ if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) {
+ BucketLayout bucketLayout = ctx.getBucketLayout(
+ keyArgs.getVolumeName(), keyArgs.getBucketName());
+ bucketLayout.validateSupportedOperation();
+ }
+ }
+ return req;
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java
index 00b8636226..0644e1f274 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java
@@ -357,4 +357,33 @@ public class S3MultipartUploadCommitPartRequest extends OMKeyRequest {
}
return req;
}
+
+ /**
+ * Validates S3 MPU commit part requests.
+ * We do not want to allow older clients to commit MPU keys to buckets which
+ * use non LEGACY layouts.
+ *
+ * @param req - the request to validate
+ * @param ctx - the validation context
+ * @return the validated request
+ * @throws OMException if the request is invalid
+ */
+ @RequestFeatureValidator(
+ conditions = ValidationCondition.OLDER_CLIENT_REQUESTS,
+ processingPhase = RequestProcessingPhase.PRE_PROCESS,
+ requestType = Type.CommitMultiPartUpload
+ )
+ public static OMRequest blockMPUCommitWithBucketLayoutFromOldClient(
+ OMRequest req, ValidationContext ctx) throws IOException {
+ if (req.getCommitMultiPartUploadRequest().hasKeyArgs()) {
+ KeyArgs keyArgs = req.getCommitMultiPartUploadRequest().getKeyArgs();
+
+ if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) {
+ BucketLayout bucketLayout = ctx.getBucketLayout(
+ keyArgs.getVolumeName(), keyArgs.getBucketName());
+ bucketLayout.validateSupportedOperation();
+ }
+ }
+ return req;
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
index 90a51180e8..ba73cde110 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
@@ -596,4 +596,33 @@ public class S3MultipartUploadCompleteRequest extends OMKeyRequest {
}
return req;
}
+
+ /**
+ * Validates S3 MPU complete requests.
+ * We do not want to allow older clients to upload MPU keys to buckets which
+ * use non LEGACY layouts.
+ *
+ * @param req - the request to validate
+ * @param ctx - the validation context
+ * @return the validated request
+ * @throws OMException if the request is invalid
+ */
+ @RequestFeatureValidator(
+ conditions = ValidationCondition.OLDER_CLIENT_REQUESTS,
+ processingPhase = RequestProcessingPhase.PRE_PROCESS,
+ requestType = Type.CompleteMultiPartUpload
+ )
+ public static OMRequest blockMPUCompleteWithBucketLayoutFromOldClient(
+ OMRequest req, ValidationContext ctx) throws IOException {
+ if (req.getCompleteMultiPartUploadRequest().hasKeyArgs()) {
+ KeyArgs keyArgs = req.getCompleteMultiPartUploadRequest().getKeyArgs();
+
+ if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) {
+ BucketLayout bucketLayout = ctx.getBucketLayout(
+ keyArgs.getVolumeName(), keyArgs.getBucketName());
+ bucketLayout.validateSupportedOperation();
+ }
+ }
+ return req;
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java
index e2e127a4fa..feb3b299ed 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java
@@ -97,7 +97,7 @@ public class RequestValidations {
m.getName(), m.getDeclaringClass().getPackage().getName(),
m.getDeclaringClass().getSimpleName());
validatedResponse =
- (OMResponse) m.invoke(null, request, response, context);
+ (OMResponse) m.invoke(null, request, validatedResponse, context);
}
} catch (InvocationTargetException | IllegalAccessException e) {
throw new ServiceException(e);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index ecec94592d..0e882a6f41 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@ -19,10 +19,13 @@ package org.apache.hadoop.ozone.protocolPB;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
import java.util.stream.Collectors;
import com.google.protobuf.ServiceException;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.UpgradeFinalizationStatus;
@@ -44,6 +47,7 @@ import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.ratis.OzoneManagerDoubleBuffer;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils;
import org.apache.hadoop.ozone.om.request.OMClientRequest;
@@ -433,6 +437,39 @@ public class OzoneManagerRequestHandler implements RequestHandler {
return resp;
}
+ @RequestFeatureValidator(
+ conditions = ValidationCondition.OLDER_CLIENT_REQUESTS,
+ processingPhase = RequestProcessingPhase.POST_PROCESS,
+ requestType = Type.LookupKey
+ )
+ public static OMResponse disallowLookupKeyWithBucketLayout(
+ OMRequest req, OMResponse resp, ValidationContext ctx)
+ throws ServiceException, IOException {
+ if (!resp.hasLookupKeyResponse()) {
+ return resp;
+ }
+ KeyInfo keyInfo = resp.getLookupKeyResponse().getKeyInfo();
+ // If the key is present inside a bucket using a non LEGACY bucket layout,
+ // then the client needs to be upgraded before proceeding.
+ if (keyInfo.hasVolumeName() && keyInfo.hasBucketName() &&
+ !ctx.getBucketLayout(keyInfo.getVolumeName(), keyInfo.getBucketName())
+ .equals(BucketLayout.LEGACY)) {
+ resp = resp.toBuilder()
+ .setStatus(Status.NOT_SUPPORTED_OPERATION)
+ .setMessage("Key is present inside a bucket with bucket layout " +
+ "features, which the client can not understand. Please upgrade" +
+ " the client to a compatible version before trying to read the" +
+ " key info for "
+ + req.getLookupKeyRequest().getKeyArgs().getVolumeName()
+ + "/" + req.getLookupKeyRequest().getKeyArgs().getBucketName()
+ + "/" + req.getLookupKeyRequest().getKeyArgs().getKeyName()
+ + ".")
+ .clearLookupKeyResponse()
+ .build();
+ }
+ return resp;
+ }
+
private ListBucketsResponse listBuckets(ListBucketsRequest request)
throws IOException {
ListBucketsResponse.Builder resp =
@@ -495,6 +532,47 @@ public class OzoneManagerRequestHandler implements RequestHandler {
return resp;
}
+ @RequestFeatureValidator(
+ conditions = ValidationCondition.OLDER_CLIENT_REQUESTS,
+ processingPhase = RequestProcessingPhase.POST_PROCESS,
+ requestType = Type.ListKeys
+ )
+ public static OMResponse disallowListKeysWithBucketLayout(
+ OMRequest req, OMResponse resp, ValidationContext ctx)
+ throws ServiceException, IOException {
+ if (!resp.hasListKeysResponse()) {
+ return resp;
+ }
+
+ // Put volume and bucket pairs into a set to avoid duplicates.
+ HashSet<Pair<String, String>> volumeBucketSet = new HashSet<>();
+ List<KeyInfo> keys = resp.getListKeysResponse().getKeyInfoList();
+ for (KeyInfo key : keys) {
+ if (key.hasVolumeName() && key.hasBucketName()) {
+ volumeBucketSet.add(
+ new ImmutablePair<>(key.getVolumeName(), key.getBucketName()));
+ }
+ }
+
+ for (Pair<String, String> volumeBucket : volumeBucketSet) {
+ // If any of the buckets have a non legacy layout, then the client is
+ // not compatible with the response.
+ if (!ctx.getBucketLayout(volumeBucket.getLeft(), volumeBucket.getRight())
+ .isLegacy()) {
+ resp = resp.toBuilder()
+ .setStatus(Status.NOT_SUPPORTED_OPERATION)
+ .setMessage("The list of keys contains keys present inside bucket" +
+ " with bucket layout features, hence the client is not able " +
+ "to understand all the keys returned. Please upgrade the"
+ + " client to get the list of keys.")
+ .clearListKeysResponse()
+ .build();
+ break;
+ }
+ }
+ return resp;
+ }
+
private ListTrashResponse listTrash(ListTrashRequest request,
int clientVersion) throws IOException {
@@ -545,6 +623,51 @@ public class OzoneManagerRequestHandler implements RequestHandler {
return resp;
}
+ @RequestFeatureValidator(
+ conditions = ValidationCondition.OLDER_CLIENT_REQUESTS,
+ processingPhase = RequestProcessingPhase.POST_PROCESS,
+ requestType = Type.ListTrash
+ )
+ public static OMResponse disallowListTrashWithBucketLayout(
+ OMRequest req, OMResponse resp, ValidationContext ctx)
+ throws ServiceException, IOException {
+ if (!resp.hasListTrashResponse()) {
+ return resp;
+ }
+
+ // Add the volume and bucket pairs to a set to avoid duplicates.
+ List<RepeatedKeyInfo> repeatedKeys =
+ resp.getListTrashResponse().getDeletedKeysList();
+ HashSet<Pair<String, String>> volumeBucketSet = new HashSet<>();
+
+ for (RepeatedKeyInfo repeatedKey : repeatedKeys) {
+ for (KeyInfo key : repeatedKey.getKeyInfoList()) {
+ if (key.hasVolumeName() && key.hasBucketName()) {
+ volumeBucketSet.add(
+ new ImmutablePair<>(key.getVolumeName(), key.getBucketName()));
+ }
+ }
+ }
+
+ // If any of the keys is present inside a bucket using a non LEGACY bucket
+ // layout, then the client needs to be upgraded before proceeding.
+ for (Pair<String, String> volumeBucket : volumeBucketSet) {
+ if (!ctx.getBucketLayout(volumeBucket.getLeft(), volumeBucket.getRight())
+ .isLegacy()) {
+ resp = resp.toBuilder()
+ .setStatus(Status.NOT_SUPPORTED_OPERATION)
+ .setMessage("The list of keys contains keys present in buckets " +
+ " using bucket layout features, hence the client is not able to"
+ + " understand all the keys returned. Please upgrade the"
+ + " client to get the list of keys.")
+ .clearListTrashResponse()
+ .build();
+ break;
+ }
+ }
+ return resp;
+ }
+
private ServiceListResponse getServiceList(ServiceListRequest request)
throws IOException {
ServiceListResponse.Builder resp = ServiceListResponse.newBuilder();
@@ -701,6 +824,40 @@ public class OzoneManagerRequestHandler implements RequestHandler {
return resp;
}
+ @RequestFeatureValidator(
+ conditions = ValidationCondition.OLDER_CLIENT_REQUESTS,
+ processingPhase = RequestProcessingPhase.POST_PROCESS,
+ requestType = Type.GetFileStatus
+ )
+ public static OMResponse disallowGetFileStatusWithBucketLayout(
+ OMRequest req, OMResponse resp, ValidationContext ctx)
+ throws ServiceException, IOException {
+ if (!resp.hasGetFileStatusResponse()) {
+ return resp;
+ }
+
+ // If the File is present inside a bucket with non LEGACY layout,
+ // then the client should be upgraded before proceeding.
+ KeyInfo keyInfo = resp.getGetFileStatusResponse().getStatus().getKeyInfo();
+ if (keyInfo.hasVolumeName() && keyInfo.hasBucketName() &&
+ !ctx.getBucketLayout(keyInfo.getVolumeName(), keyInfo.getBucketName())
+ .isLegacy()) {
+ resp = resp.toBuilder()
+ .setStatus(Status.NOT_SUPPORTED_OPERATION)
+ .setMessage("Key is present in a bucket using bucket layout features"
+ + " which the client can not understand."
+ + " Please upgrade the client before trying to read the key info"
+ + " for "
+ + req.getGetFileStatusRequest().getKeyArgs().getVolumeName()
+ + "/" + req.getGetFileStatusRequest().getKeyArgs().getBucketName()
+ + "/" + req.getGetFileStatusRequest().getKeyArgs().getKeyName()
+ + ".")
+ .clearGetFileStatusResponse()
+ .build();
+ }
+ return resp;
+ }
+
private LookupFileResponse lookupFile(LookupFileRequest request,
int clientVersion) throws IOException {
KeyArgs keyArgs = request.getKeyArgs();
@@ -745,6 +902,37 @@ public class OzoneManagerRequestHandler implements RequestHandler {
return resp;
}
+ @RequestFeatureValidator(
+ conditions = ValidationCondition.OLDER_CLIENT_REQUESTS,
+ processingPhase = RequestProcessingPhase.POST_PROCESS,
+ requestType = Type.LookupFile
+ )
+ public static OMResponse disallowLookupFileWithBucketLayout(
+ OMRequest req, OMResponse resp, ValidationContext ctx)
+ throws ServiceException, IOException {
+ if (!resp.hasLookupFileResponse()) {
+ return resp;
+ }
+ KeyInfo keyInfo = resp.getLookupFileResponse().getKeyInfo();
+ if (keyInfo.hasVolumeName() && keyInfo.hasBucketName() &&
+ !ctx.getBucketLayout(keyInfo.getVolumeName(), keyInfo.getBucketName())
+ .equals(BucketLayout.LEGACY)) {
+ resp = resp.toBuilder()
+ .setStatus(Status.NOT_SUPPORTED_OPERATION)
+ .setMessage("File is present inside a bucket with bucket layout " +
+ "features, which the client can not understand. Please upgrade" +
+ " the client to a compatible version before trying to read the" +
+ " key info for "
+ + req.getLookupFileRequest().getKeyArgs().getVolumeName()
+ + "/" + req.getLookupFileRequest().getKeyArgs().getBucketName()
+ + "/" + req.getLookupFileRequest().getKeyArgs().getKeyName()
+ + ".")
+ .clearLookupFileResponse()
+ .build();
+ }
+ return resp;
+ }
+
private ListStatusResponse listStatus(
ListStatusRequest request, int clientVersion) throws IOException {
KeyArgs keyArgs = request.getKeyArgs();
@@ -796,6 +984,52 @@ public class OzoneManagerRequestHandler implements RequestHandler {
return resp;
}
+ @RequestFeatureValidator(
+ conditions = ValidationCondition.OLDER_CLIENT_REQUESTS,
+ processingPhase = RequestProcessingPhase.POST_PROCESS,
+ requestType = Type.ListStatus
+ )
+ public static OMResponse disallowListStatusResponseWithBucketLayout(
+ OMRequest req, OMResponse resp, ValidationContext ctx)
+ throws ServiceException, IOException {
+ if (!resp.hasListStatusResponse()) {
+ return resp;
+ }
+
+ // Add the volume and bucket pairs to a set to avoid duplicate entries.
+ List<OzoneFileStatusProto> statuses =
+ resp.getListStatusResponse().getStatusesList();
+ HashSet<Pair<String, String>> volumeBucketSet = new HashSet<>();
+
+ for (OzoneFileStatusProto status : statuses) {
+ KeyInfo keyInfo = status.getKeyInfo();
+ if (keyInfo.hasVolumeName() && keyInfo.hasBucketName()) {
+ volumeBucketSet.add(
+ new ImmutablePair<>(keyInfo.getVolumeName(),
+ keyInfo.getBucketName()));
+ }
+ }
+
+ // If any of the keys are present in a bucket with a non LEGACY bucket
+ // layout, then the client needs to be upgraded before proceeding.
+ for (Pair<String, String> volumeBucket : volumeBucketSet) {
+ if (!ctx.getBucketLayout(volumeBucket.getLeft(),
+ volumeBucket.getRight()).isLegacy()) {
+ resp = resp.toBuilder()
+ .setStatus(Status.NOT_SUPPORTED_OPERATION)
+ .setMessage("The list of keys is present in a bucket using bucket"
+ + " layout features, hence the client is not able to"
+ + " represent all the keys returned."
+ + " Please upgrade the client to get the list of keys.")
+ .clearListStatusResponse()
+ .build();
+ break;
+ }
+ }
+
+ return resp;
+ }
+
private FinalizeUpgradeProgressResponse reportUpgradeProgress(
FinalizeUpgradeProgressRequest request) throws IOException {
String upgradeClientId = request.getUpgradeClientId();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org