You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ra...@apache.org on 2021/04/07 03:25:50 UTC
[ozone] 23/29: HDDS-4924. [FSO]S3Multipart: Implement
OzoneBucket#listParts (#2016)
This is an automated email from the ASF dual-hosted git repository.
rakeshr pushed a commit to branch HDDS-2939
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 2b657cd4d5800a57ca62964aacbe0fd009532ee5
Author: Rakesh Radhakrishnan <ra...@apache.org>
AuthorDate: Wed Mar 10 22:04:39 2021 +0530
HDDS-4924. [FSO]S3Multipart: Implement OzoneBucket#listParts (#2016)
---
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 2 +-
.../rpc/TestOzoneClientMultipartUploadV1.java | 265 +++++++++++++++++++++
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 58 ++++-
.../S3MultipartUploadCompleteRequest.java | 1 -
4 files changed, 321 insertions(+), 5 deletions(-)
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
index c63c21f..f1f6454 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
@@ -178,7 +178,7 @@ public final class OzoneFSUtils {
if (fileName != null) {
return fileName.toString();
}
- // failed to find a parent directory.
+ // no parent directory.
return "";
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadV1.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadV1.java
index 76feec8..0e981d6 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadV1.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadV1.java
@@ -18,15 +18,18 @@
package org.apache.hadoop.ozone.client.rpc;
import org.apache.commons.lang3.RandomUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.OzoneTestUtils;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
@@ -48,6 +51,7 @@ import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.request.TestOMRequestUtils;
import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -58,9 +62,11 @@ import org.junit.rules.Timeout;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.UUID;
@@ -566,6 +572,265 @@ public class TestOzoneClientMultipartUploadV1 {
// not making any assertion for the same.
}
+ @Test
+ public void testListMultipartUploadParts() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String parentDir = "a/b/c/d/e/f/";
+ String keyName = parentDir + "file-ABC";
+
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ Map<Integer, String> partsMap = new TreeMap<>();
+ String uploadID = initiateMultipartUpload(bucket, keyName, STAND_ALONE,
+ ONE);
+ String partName1 = uploadPart(bucket, keyName, uploadID, 1,
+ generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte)97));
+ partsMap.put(1, partName1);
+
+ String partName2 =uploadPart(bucket, keyName, uploadID, 2,
+ generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte)97));
+ partsMap.put(2, partName2);
+
+ String partName3 =uploadPart(bucket, keyName, uploadID, 3,
+ generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte)97));
+ partsMap.put(3, partName3);
+
+ OzoneMultipartUploadPartListParts ozoneMultipartUploadPartListParts =
+ bucket.listParts(keyName, uploadID, 0, 3);
+
+ Assert.assertEquals(STAND_ALONE,
+ ozoneMultipartUploadPartListParts.getReplicationType());
+ Assert.assertEquals(3,
+ ozoneMultipartUploadPartListParts.getPartInfoList().size());
+
+ verifyPartNamesInDB(volumeName, bucketName, parentDir, keyName, partsMap,
+ ozoneMultipartUploadPartListParts, uploadID);
+
+ Assert.assertFalse(ozoneMultipartUploadPartListParts.isTruncated());
+ }
+
+ private void verifyPartNamesInDB(String volumeName, String bucketName,
+ String parentDir, String keyName, Map<Integer, String> partsMap,
+ OzoneMultipartUploadPartListParts ozoneMultipartUploadPartListParts,
+ String uploadID) throws IOException {
+
+ List<String> listPartNames = new ArrayList<>();
+ String keyPartName = verifyPartNames(partsMap, 0,
+ ozoneMultipartUploadPartListParts);
+ listPartNames.add(keyPartName);
+
+ keyPartName = verifyPartNames(partsMap, 1,
+ ozoneMultipartUploadPartListParts);
+ listPartNames.add(keyPartName);
+
+ keyPartName = verifyPartNames(partsMap, 2,
+ ozoneMultipartUploadPartListParts);
+ listPartNames.add(keyPartName);
+
+ OMMetadataManager metadataMgr =
+ cluster.getOzoneManager().getMetadataManager();
+ String multipartKey = getMultipartKey(uploadID, volumeName, bucketName,
+ keyName, metadataMgr);
+ OmMultipartKeyInfo omMultipartKeyInfo =
+ metadataMgr.getMultipartInfoTable().get(multipartKey);
+ Assert.assertNotNull(omMultipartKeyInfo);
+
+ long parentID = getParentID(volumeName, bucketName, keyName, metadataMgr);
+ TreeMap<Integer, OzoneManagerProtocolProtos.PartKeyInfo> partKeyInfoMap =
+ omMultipartKeyInfo.getPartKeyInfoMap();
+ for (Map.Entry<Integer, OzoneManagerProtocolProtos.PartKeyInfo> entry :
+ partKeyInfoMap.entrySet()) {
+ OzoneManagerProtocolProtos.PartKeyInfo partKeyInfo = entry.getValue();
+ String partKeyName = partKeyInfo.getPartName();
+
+ // partKeyName format in DB - <parentID>/partFileName + ClientID
+ Assert.assertTrue("Invalid partKeyName format in DB",
+ partKeyName.startsWith(parentID + OzoneConsts.OM_KEY_PREFIX));
+ partKeyName = StringUtils.remove(partKeyName,
+ parentID + OzoneConsts.OM_KEY_PREFIX);
+
+ // reconstruct full part name with volume, bucket, partKeyName
+ String fullKeyPartName = metadataMgr.getOzoneKey(volumeName, bucketName,
+ parentDir + partKeyName);
+
+ listPartNames.remove(fullKeyPartName);
+ }
+
+ Assert.assertTrue("Wrong partKeyName format in DB!",
+ listPartNames.isEmpty());
+ }
+
+ private String verifyPartNames(Map<Integer, String> partsMap, int index,
+ OzoneMultipartUploadPartListParts ozoneMultipartUploadPartListParts) {
+
+ Assert.assertEquals(partsMap.get(ozoneMultipartUploadPartListParts
+ .getPartInfoList().get(index).getPartNumber()),
+ ozoneMultipartUploadPartListParts.getPartInfoList().get(index)
+ .getPartName());
+
+ return ozoneMultipartUploadPartListParts.getPartInfoList().get(index)
+ .getPartName();
+ }
+
+ @Test
+ public void testListMultipartUploadPartsWithContinuation()
+ throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String keyName = UUID.randomUUID().toString();
+
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ Map<Integer, String> partsMap = new TreeMap<>();
+ String uploadID = initiateMultipartUpload(bucket, keyName, STAND_ALONE,
+ ONE);
+ String partName1 = uploadPart(bucket, keyName, uploadID, 1,
+ generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte)97));
+ partsMap.put(1, partName1);
+
+ String partName2 =uploadPart(bucket, keyName, uploadID, 2,
+ generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte)97));
+ partsMap.put(2, partName2);
+
+ String partName3 =uploadPart(bucket, keyName, uploadID, 3,
+ generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte)97));
+ partsMap.put(3, partName3);
+
+ OzoneMultipartUploadPartListParts ozoneMultipartUploadPartListParts =
+ bucket.listParts(keyName, uploadID, 0, 2);
+
+ Assert.assertEquals(STAND_ALONE,
+ ozoneMultipartUploadPartListParts.getReplicationType());
+
+ Assert.assertEquals(2,
+ ozoneMultipartUploadPartListParts.getPartInfoList().size());
+
+ Assert.assertEquals(partsMap.get(ozoneMultipartUploadPartListParts
+ .getPartInfoList().get(0).getPartNumber()),
+ ozoneMultipartUploadPartListParts.getPartInfoList().get(0)
+ .getPartName());
+ Assert.assertEquals(partsMap.get(ozoneMultipartUploadPartListParts
+ .getPartInfoList().get(1).getPartNumber()),
+ ozoneMultipartUploadPartListParts.getPartInfoList().get(1)
+ .getPartName());
+
+ // Get remaining
+ Assert.assertTrue(ozoneMultipartUploadPartListParts.isTruncated());
+ ozoneMultipartUploadPartListParts = bucket.listParts(keyName, uploadID,
+ ozoneMultipartUploadPartListParts.getNextPartNumberMarker(), 2);
+
+ Assert.assertEquals(1,
+ ozoneMultipartUploadPartListParts.getPartInfoList().size());
+ Assert.assertEquals(partsMap.get(ozoneMultipartUploadPartListParts
+ .getPartInfoList().get(0).getPartNumber()),
+ ozoneMultipartUploadPartListParts.getPartInfoList().get(0)
+ .getPartName());
+
+
+ // As we don't have any parts for this, we should get false here
+ Assert.assertFalse(ozoneMultipartUploadPartListParts.isTruncated());
+
+ }
+
+ @Test
+ public void testListPartsInvalidPartMarker() throws Exception {
+ try {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String keyName = UUID.randomUUID().toString();
+
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ bucket.listParts(keyName, "random", -1, 2);
+ Assert.fail("Should throw exception as partNumber is an invalid number!");
+ } catch (IllegalArgumentException ex) {
+ GenericTestUtils.assertExceptionContains("Should be greater than or "
+ + "equal to zero", ex);
+ }
+ }
+
+ @Test
+ public void testListPartsInvalidMaxParts() throws Exception {
+ try {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String keyName = UUID.randomUUID().toString();
+
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ bucket.listParts(keyName, "random", 1, -1);
+ Assert.fail("Should throw exception as max parts is an invalid number!");
+ } catch (IllegalArgumentException ex) {
+ GenericTestUtils.assertExceptionContains("Max Parts Should be greater "
+ + "than zero", ex);
+ }
+ }
+
+ @Test
+ public void testListPartsWithPartMarkerGreaterThanPartCount()
+ throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String keyName = UUID.randomUUID().toString();
+
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+
+ String uploadID = initiateMultipartUpload(bucket, keyName, STAND_ALONE,
+ ONE);
+ uploadPart(bucket, keyName, uploadID, 1,
+ generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte)97));
+
+
+ OzoneMultipartUploadPartListParts ozoneMultipartUploadPartListParts =
+ bucket.listParts(keyName, uploadID, 100, 2);
+
+ // Should return empty
+
+ Assert.assertEquals(0,
+ ozoneMultipartUploadPartListParts.getPartInfoList().size());
+ Assert.assertEquals(STAND_ALONE,
+ ozoneMultipartUploadPartListParts.getReplicationType());
+
+ // As we don't have any parts with greater than partNumberMarker and list
+ // is not truncated, so it should return false here.
+ Assert.assertFalse(ozoneMultipartUploadPartListParts.isTruncated());
+
+ }
+
+ @Test
+ public void testListPartsWithInvalidUploadID() throws Exception {
+ OzoneTestUtils
+ .expectOmException(NO_SUCH_MULTIPART_UPLOAD_ERROR, () -> {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String keyName = UUID.randomUUID().toString();
+
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+ OzoneMultipartUploadPartListParts ozoneMultipartUploadPartListParts =
+ bucket.listParts(keyName, "random", 100, 2);
+ });
+ }
+
private String verifyUploadedPart(String volumeName, String bucketName,
String keyName, String uploadID, String partName,
OMMetadataManager metadataMgr) throws IOException {
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index 24afc5f..f45a96b 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -1416,8 +1416,8 @@ public class KeyManagerImpl implements KeyManager {
metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName,
bucketName);
try {
- String multipartKey = metadataManager.getMultipartKey(volumeName,
- bucketName, keyName, uploadID);
+ String multipartKey = getMultipartKey(volumeName, bucketName,
+ keyName, uploadID);
OmMultipartKeyInfo multipartKeyInfo =
metadataManager.getMultipartInfoTable().get(multipartKey);
@@ -1445,8 +1445,10 @@ public class KeyManagerImpl implements KeyManager {
// than part number marker
if (partKeyInfoEntry.getKey() > partNumberMarker) {
PartKeyInfo partKeyInfo = partKeyInfoEntry.getValue();
+ String partName = getPartName(partKeyInfo, volumeName, bucketName,
+ keyName);
OmPartInfo omPartInfo = new OmPartInfo(partKeyInfo.getPartNumber(),
- partKeyInfo.getPartName(),
+ partName,
partKeyInfo.getPartKeyInfo().getModificationTime(),
partKeyInfo.getPartKeyInfo().getDataSize());
omPartInfoList.add(omPartInfo);
@@ -1506,6 +1508,56 @@ public class KeyManagerImpl implements KeyManager {
}
}
+ private String getPartName(PartKeyInfo partKeyInfo, String volName,
+ String buckName, String keyName) {
+
+ String partName = partKeyInfo.getPartName();
+
+ if (OzoneManagerRatisUtils.isBucketFSOptimized()) {
+ String parentDir = OzoneFSUtils.getParentDir(keyName);
+ String partFileName = OzoneFSUtils.getFileName(partKeyInfo.getPartName());
+
+ StringBuilder fullKeyPartName = new StringBuilder();
+ fullKeyPartName.append(OZONE_URI_DELIMITER);
+ fullKeyPartName.append(volName);
+ fullKeyPartName.append(OZONE_URI_DELIMITER);
+ fullKeyPartName.append(buckName);
+ if (StringUtils.isNotEmpty(parentDir)) {
+ fullKeyPartName.append(OZONE_URI_DELIMITER);
+ fullKeyPartName.append(parentDir);
+ }
+ fullKeyPartName.append(OZONE_URI_DELIMITER);
+ fullKeyPartName.append(partFileName);
+
+ return fullKeyPartName.toString();
+ }
+ return partName;
+ }
+
+ private String getMultipartKey(String volumeName, String bucketName,
+ String keyName, String uploadID) throws IOException {
+
+ if (OzoneManagerRatisUtils.isBucketFSOptimized()) {
+ OMMetadataManager metaMgr = ozoneManager.getMetadataManager();
+ String fileName = OzoneFSUtils.getFileName(keyName);
+ Iterator<Path> pathComponents = Paths.get(keyName).iterator();
+ String bucketKey = metaMgr.getBucketKey(volumeName, bucketName);
+ OmBucketInfo omBucketInfo =
+ metaMgr.getBucketTable().get(bucketKey);
+ long bucketId = omBucketInfo.getObjectID();
+ long parentID = OMFileRequest.getParentID(bucketId, pathComponents,
+ keyName, metaMgr);
+
+ String multipartKey = metaMgr.getMultipartKey(parentID, fileName,
+ uploadID);
+
+ return multipartKey;
+ } else {
+ return metadataManager.getMultipartKey(volumeName,
+ bucketName, keyName, uploadID);
+ }
+ }
+
/**
* Add acl for Ozone object. Return true if acl is added successfully else
* false.
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
index 162cf2f..8fcf992 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
-import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.om.request.key.OMKeyRequest;
import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org