You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ra...@apache.org on 2021/12/23 12:43:14 UTC
[ozone] branch master updated: HDDS-5807. [FSO] Merge HDDS-4653 changes to support TDE for MPU Keys in FSO bucket layout. (#2941)
This is an automated email from the ASF dual-hosted git repository.
rakeshr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 9dc63a3 HDDS-5807. [FSO] Merge HDDS-4653 changes to support TDE for MPU Keys in FSO bucket layout. (#2941)
9dc63a3 is described below
commit 9dc63a3dae9d423cd0fd669aae3277d3fffb599d
Author: Jyotinder Singh <jy...@gmail.com>
AuthorDate: Thu Dec 23 18:09:10 2021 +0530
HDDS-5807. [FSO] Merge HDDS-4653 changes to support TDE for MPU Keys in FSO bucket layout. (#2941)
---
.../client/rpc/TestOzoneAtRestEncryption.java | 27 +++++++++++++++--
.../ozone/om/request/file/OMFileRequest.java | 25 ++++++++++++++++
.../om/request/key/OMKeyCreateRequestWithFSO.java | 27 +++++++++++++++++
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 35 +++++++++++++++++++---
.../S3InitiateMultipartUploadRequestWithFSO.java | 15 ++--------
.../S3MultipartUploadCompleteRequest.java | 7 -----
.../S3MultipartUploadCompleteRequestWithFSO.java | 15 +---------
7 files changed, 111 insertions(+), 40 deletions(-)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java
index 89d502c..d5e60d2 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java
@@ -31,6 +31,7 @@ import java.util.Map;
import java.util.Random;
import java.util.TreeMap;
import java.util.UUID;
+import java.util.Collection;
import com.google.common.cache.Cache;
import org.apache.hadoop.conf.StorageUnit;
@@ -68,6 +69,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartCommitUploadPartInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.ozone.test.GenericTestUtils;
import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
@@ -78,13 +80,23 @@ import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
import org.mockito.Mockito;
/**
* This class is to test all the public facing APIs of Ozone Client.
*/
+@RunWith(Parameterized.class)
public class TestOzoneAtRestEncryption {
+ @Parameterized.Parameters
+ public static Collection<BucketLayout> data() {
+ return Arrays.asList(
+ BucketLayout.FILE_SYSTEM_OPTIMIZED,
+ BucketLayout.OBJECT_STORE);
+ }
+
private static MiniOzoneCluster cluster = null;
private static MiniKMS miniKMS;
private static OzoneClient ozClient = null;
@@ -106,6 +118,11 @@ public class TestOzoneAtRestEncryption {
private static final int DEFAULT_CRYPTO_BUFFER_SIZE = 8 * 1024; // 8KB
// (this is the default Crypto Buffer size as determined by the config
// hadoop.security.crypto.buffer.size)
+ private final BucketLayout bucketLayout;
+
+ public TestOzoneAtRestEncryption(BucketLayout layout) {
+ bucketLayout = layout;
+ }
@BeforeClass
public static void init() throws Exception {
@@ -186,6 +203,7 @@ public class TestOzoneAtRestEncryption {
store.createVolume(volumeName);
OzoneVolume volume = store.getVolume(volumeName);
BucketArgs bucketArgs = BucketArgs.newBuilder()
+ .setBucketLayout(bucketLayout)
.setBucketEncryptionKey(TEST_KEY).build();
volume.createBucket(bucketName, bucketArgs);
OzoneBucket bucket = volume.getBucket(bucketName);
@@ -253,7 +271,8 @@ public class TestOzoneAtRestEncryption {
store.createVolume(volumeName);
OzoneVolume volume = store.getVolume(volumeName);
BucketArgs bucketArgs = BucketArgs.newBuilder()
- .setBucketEncryptionKey(TEST_KEY).build();
+ .setBucketEncryptionKey(TEST_KEY)
+ .setBucketLayout(bucketLayout).build();
volume.createBucket(bucketName, bucketArgs);
return volume.getBucket(bucketName);
}
@@ -263,7 +282,8 @@ public class TestOzoneAtRestEncryption {
store.createVolume(linkVol);
OzoneVolume linkVolume = store.getVolume(linkVol);
BucketArgs linkBucketArgs = BucketArgs.newBuilder()
- .setSourceVolume(sourceVol).setSourceBucket(sourceBucket).build();
+ .setSourceVolume(sourceVol).setSourceBucket(sourceBucket)
+ .setBucketLayout(bucketLayout).build();
linkVolume.createBucket(linkBucket, linkBucketArgs);
return linkVolume.getBucket(linkBucket);
}
@@ -288,7 +308,8 @@ public class TestOzoneAtRestEncryption {
//Bucket with Encryption & GDPR enforced
BucketArgs bucketArgs = BucketArgs.newBuilder()
.setBucketEncryptionKey(TEST_KEY)
- .addMetadata(OzoneConsts.GDPR_FLAG, "true").build();
+ .addMetadata(OzoneConsts.GDPR_FLAG, "true")
+ .setBucketLayout(bucketLayout).build();
volume.createBucket(bucketName, bucketArgs);
OzoneBucket bucket = volume.getBucket(bucketName);
Assert.assertEquals(bucketName, bucket.getName());
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java
index 38313b8..9f05fee 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java
@@ -1012,6 +1012,31 @@ public final class OMFileRequest {
}
/**
+ * Get parent ID for the user given keyName.
+ *
+ * @param omMetadataManager
+ * @param volumeName - volume name.
+ * @param bucketName - bucket name.
+ * @param keyName - key name.
+ * @return
+ * @throws IOException
+ */
+ public static long getParentId(OMMetadataManager omMetadataManager,
+ String volumeName, String bucketName,
+ String keyName)
+ throws IOException {
+
+ String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
+ OmBucketInfo omBucketInfo =
+ omMetadataManager.getBucketTable().get(bucketKey);
+
+ long bucketId = omBucketInfo.getObjectID();
+ Iterator<Path> pathComponents = Paths.get(keyName).iterator();
+ return OMFileRequest
+ .getParentID(bucketId, pathComponents, keyName, omMetadataManager);
+ }
+
+ /**
* Validates volume and bucket existence.
*
* @param metadataManager
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestWithFSO.java
index 6aa79de..fb1dd04 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestWithFSO.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestWithFSO.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
@@ -53,6 +54,7 @@ import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_
import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK;
import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.OMDirectoryResult.DIRECTORY_EXISTS;
import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.OMDirectoryResult.FILE_EXISTS_IN_GIVENPATH;
+import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.getParentId;
/**
* Handles CreateKey request layout version1.
@@ -232,4 +234,29 @@ public class OMKeyCreateRequestWithFSO extends OMKeyCreateRequest {
return omClientResponse;
}
+
+ /**
+ * Returns the DB key name of a multipart open key in OM metadata store.
+ *
+ * @param volumeName - volume name.
+ * @param bucketName - bucket name.
+ * @param keyName - key name.
+ * @param uploadID - Multi part upload ID for this key.
+ * @param omMetadataManager
+ * @return
+ * @throws IOException
+ */
+ @Override
+ protected String getDBMultipartOpenKey(String volumeName, String bucketName,
+ String keyName, String uploadID,
+ OMMetadataManager omMetadataManager)
+ throws IOException {
+
+ long parentId =
+ getParentId(omMetadataManager, volumeName, bucketName, keyName);
+
+ String fileName = OzoneFSUtils.getFileName(keyName);
+
+ return omMetadataManager.getMultipartKey(parentId, fileName, uploadID);
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
index 92a9c07..b4528b8 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
@@ -467,11 +467,18 @@ public abstract class OMKeyRequest extends OMClientRequest {
try {
ResolvedBucket resolvedBucket = ozoneManager.resolveBucketLink(
Pair.of(keyArgs.getVolumeName(), keyArgs.getBucketName()));
+
+ // Get the DB key name for looking up keyInfo in OpenKeyTable with
+ // resolved volume/bucket.
+ String dbMultipartOpenKey =
+ getDBMultipartOpenKey(resolvedBucket.realVolume(),
+ resolvedBucket.realBucket(), keyArgs.getKeyName(),
+ keyArgs.getMultipartUploadID(), omMetadataManager);
+
OmKeyInfo omKeyInfo =
- omMetadataManager.getOpenKeyTable(getBucketLayout()).get(
- omMetadataManager.getMultipartKey(resolvedBucket.realVolume(),
- resolvedBucket.realBucket(), keyArgs.getKeyName(),
- keyArgs.getMultipartUploadID()));
+ omMetadataManager.getOpenKeyTable(getBucketLayout())
+ .get(dbMultipartOpenKey);
+
if (omKeyInfo != null && omKeyInfo.getFileEncryptionInfo() != null) {
newKeyArgs.setFileEncryptionInfo(
OMPBHelper.convert(omKeyInfo.getFileEncryptionInfo()));
@@ -743,4 +750,24 @@ public abstract class OMKeyRequest extends OMClientRequest {
size, encInfo, prefixManager, omBucketInfo, omPathInfo,
transactionLogIndex, objectID);
}
+
+ /**
+ * Returns the DB key name of a multipart open key in OM metadata store.
+ *
+ * @param volumeName - volume name.
+ * @param bucketName - bucket name.
+ * @param keyName - key name.
+ * @param uploadID - Multi part upload ID for this key.
+ * @param omMetadataManager
+ * @return
+ * @throws IOException
+ */
+ protected String getDBMultipartOpenKey(String volumeName, String bucketName,
+ String keyName, String uploadID,
+ OMMetadataManager omMetadataManager)
+ throws IOException {
+
+ return omMetadataManager
+ .getMultipartKey(volumeName, bucketName, keyName, uploadID);
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequestWithFSO.java
index 1ef6ed2..f1abc21 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequestWithFSO.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequestWithFSO.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Multipa
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartInfoInitiateResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
import java.io.IOException;
import java.nio.file.Paths;
@@ -51,7 +52,6 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
-import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION;
import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK;
import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.OMDirectoryResult.DIRECTORY_EXISTS;
@@ -112,17 +112,6 @@ public class S3InitiateMultipartUploadRequestWithFSO
validateBucketAndVolume(omMetadataManager, volumeName, bucketName);
- // If KMS is configured and TDE is enabled on bucket, throw MPU not
- // supported.
- if (ozoneManager.getKmsProvider() != null) {
- if (omMetadataManager.getBucketTable().get(
- omMetadataManager.getBucketKey(volumeName, bucketName))
- .getEncryptionKeyInfo() != null) {
- throw new OMException("MultipartUpload is not yet supported on " +
- "encrypted buckets", NOT_SUPPORTED_OPERATION);
- }
- }
-
OMFileRequest.OMPathInfoWithFSO pathInfoFSO = OMFileRequest
.verifyDirectoryKeysInPath(omMetadataManager, volumeName, bucketName,
keyName, Paths.get(keyName));
@@ -187,6 +176,8 @@ public class S3InitiateMultipartUploadRequestWithFSO
.setAcls(OzoneAclUtil.fromProtobuf(keyArgs.getAclsList()))
.setObjectID(pathInfoFSO.getLeafNodeObjectId())
.setUpdateID(transactionLogIndex)
+ .setFileEncryptionInfo(keyArgs.hasFileEncryptionInfo() ?
+ OMPBHelper.convert(keyArgs.getFileEncryptionInfo()) : null)
.setParentObjectID(pathInfoFSO.getLastKnownParentId())
.build();
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
index 873e18c..eee2ab7 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
@@ -384,13 +384,6 @@ public class S3MultipartUploadCompleteRequest extends OMKeyRequest {
return omMetadataManager.getOzoneKey(volumeName, bucketName, keyName);
}
- protected String getDBMultipartOpenKey(String volumeName, String bucketName,
- String keyName, String uploadID, OMMetadataManager omMetadataManager)
- throws IOException {
- return omMetadataManager
- .getMultipartKey(volumeName, bucketName, keyName, uploadID);
- }
-
protected OmKeyInfo getOmKeyInfoFromKeyTable(String dbOzoneKey,
String keyName, OMMetadataManager omMetadataManager) throws IOException {
return omMetadataManager.getKeyTable(getBucketLayout()).get(dbOzoneKey);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java
index 9b43652..33b9f41 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java
@@ -22,7 +22,6 @@ import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
-import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
@@ -36,11 +35,11 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.util.Iterator;
import java.util.List;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_A_FILE;
import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.OMDirectoryResult.DIRECTORY_EXISTS;
+import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.getParentId;
/**
* Handle Multipart upload complete request.
@@ -156,17 +155,5 @@ public class S3MultipartUploadCompleteRequestWithFSO
multipartKey, dbMultipartOpenKey, omKeyInfo, unUsedParts,
getBucketLayout());
}
-
- private long getParentId(OMMetadataManager omMetadataManager,
- String volumeName, String bucketName, String keyName) throws IOException {
-
- String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
- OmBucketInfo omBucketInfo =
- omMetadataManager.getBucketTable().get(bucketKey);
- long bucketId = omBucketInfo.getObjectID();
- Iterator<Path> pathComponents = Paths.get(keyName).iterator();
- return OMFileRequest
- .getParentID(bucketId, pathComponents, keyName, omMetadataManager);
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org