You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ra...@apache.org on 2021/04/08 03:56:28 UTC
[ozone] 22/32: HDDS-4835. [FSO]S3Multipart: Implement
UploadAbortRequest (#1997)
This is an automated email from the ASF dual-hosted git repository.
rakeshr pushed a commit to branch HDDS-2939
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 46d9192a5d2f3cfd4677f02d3735a7b1bd2ca65c
Author: Rakesh Radhakrishnan <ra...@apache.org>
AuthorDate: Tue Mar 9 17:11:31 2021 +0530
HDDS-4835. [FSO]S3Multipart: Implement UploadAbortRequest (#1997)
---
.../rpc/TestOzoneClientMultipartUploadV1.java | 179 +++++++++++++++++++++
.../om/ratis/utils/OzoneManagerRatisUtils.java | 4 +
.../multipart/S3MultipartUploadAbortRequest.java | 43 +++--
.../multipart/S3MultipartUploadAbortRequestV1.java | 90 +++++++++++
.../S3MultipartUploadCommitPartRequestV1.java | 23 +--
.../S3MultipartUploadCompleteRequest.java | 17 +-
.../S3MultipartUploadCompleteRequestV1.java | 16 ++
.../S3MultipartUploadAbortResponseV1.java | 55 +++++++
.../TestS3InitiateMultipartUploadRequest.java | 24 +--
.../TestS3InitiateMultipartUploadRequestV1.java | 57 +------
.../s3/multipart/TestS3MultipartRequest.java | 7 +-
.../TestS3MultipartUploadAbortRequest.java | 29 +++-
.../TestS3MultipartUploadAbortRequestV1.java | 69 ++++++++
.../s3/multipart/TestS3MultipartResponse.java | 25 ++-
.../TestS3MultipartUploadAbortResponse.java | 22 ++-
.../TestS3MultipartUploadAbortResponseV1.java | 75 +++++++++
16 files changed, 622 insertions(+), 113 deletions(-)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadV1.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadV1.java
index 1ab2cc3..76feec8 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadV1.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadV1.java
@@ -31,7 +31,10 @@ import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartCommitUploadPartInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
@@ -39,8 +42,12 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.hadoop.hdds.StringUtils.string2Bytes;
import static org.apache.hadoop.hdds.client.ReplicationFactor.THREE;
+import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
+import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.request.TestOMRequestUtils;
+import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -49,7 +56,10 @@ import org.junit.Test;
import org.junit.rules.Timeout;
import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.Arrays;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.TreeMap;
@@ -58,6 +68,9 @@ import java.util.UUID;
import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE;
import static org.apache.hadoop.hdds.client.ReplicationType.STAND_ALONE;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
/**
* This test verifies all the S3 multipart client apis - layout version V1.
@@ -450,6 +463,172 @@ public class TestOzoneClientMultipartUploadV1 {
}
}
+ @Test
+ public void testAbortUploadFail() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String keyName = UUID.randomUUID().toString();
+
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ OzoneTestUtils.expectOmException(NO_SUCH_MULTIPART_UPLOAD_ERROR,
+ () -> bucket.abortMultipartUpload(keyName, "random"));
+ }
+
+ @Test
+ public void testAbortUploadFailWithInProgressPartUpload() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String parentDir = "a/b/c/d/";
+ String keyName = parentDir + UUID.randomUUID().toString();
+
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ OmMultipartInfo omMultipartInfo = bucket.initiateMultipartUpload(keyName,
+ STAND_ALONE, ONE);
+
+ Assert.assertNotNull(omMultipartInfo.getUploadID());
+
+ // Do not close output stream.
+ byte[] data = "data".getBytes(UTF_8);
+ OzoneOutputStream ozoneOutputStream = bucket.createMultipartKey(keyName,
+ data.length, 1, omMultipartInfo.getUploadID());
+ ozoneOutputStream.write(data, 0, data.length);
+
+ // Abort before completing part upload.
+ bucket.abortMultipartUpload(keyName, omMultipartInfo.getUploadID());
+
+ try {
+ ozoneOutputStream.close();
+ fail("testAbortUploadFailWithInProgressPartUpload failed");
+ } catch (IOException ex) {
+ assertTrue(ex instanceof OMException);
+ assertEquals(NO_SUCH_MULTIPART_UPLOAD_ERROR,
+ ((OMException) ex).getResult());
+ }
+ }
+
+ @Test
+ public void testAbortUploadSuccessWithOutAnyParts() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String parentDir = "a/b/c/d/";
+ String keyName = parentDir + UUID.randomUUID().toString();
+
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ String uploadID = initiateMultipartUpload(bucket, keyName, STAND_ALONE,
+ ONE);
+ bucket.abortMultipartUpload(keyName, uploadID);
+ }
+
+ @Test
+ public void testAbortUploadSuccessWithParts() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String parentDir = "a/b/c/d/";
+ String keyName = parentDir + UUID.randomUUID().toString();
+
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ String uploadID = initiateMultipartUpload(bucket, keyName, STAND_ALONE,
+ ONE);
+ String partName = uploadPart(bucket, keyName, uploadID, 1,
+ "data".getBytes(UTF_8));
+
+ OMMetadataManager metadataMgr =
+ cluster.getOzoneManager().getMetadataManager();
+ String multipartKey = verifyUploadedPart(volumeName, bucketName, keyName,
+ uploadID, partName, metadataMgr);
+
+ bucket.abortMultipartUpload(keyName, uploadID);
+
+ OmKeyInfo omKeyInfo = metadataMgr.getOpenKeyTable().get(multipartKey);
+ OmMultipartKeyInfo omMultipartKeyInfo =
+ metadataMgr.getMultipartInfoTable().get(multipartKey);
+ Assert.assertNull(omKeyInfo);
+ Assert.assertNull(omMultipartKeyInfo);
+
+ // Since deleteTable operation is performed via
+ // batchOp - Table.putWithBatch(), which is an async operation and
+ // not making any assertion for the same.
+ }
+
+ private String verifyUploadedPart(String volumeName, String bucketName,
+ String keyName, String uploadID, String partName,
+ OMMetadataManager metadataMgr) throws IOException {
+ String multipartKey = getMultipartKey(uploadID, volumeName, bucketName,
+ keyName, metadataMgr);
+ OmKeyInfo omKeyInfo = metadataMgr.getOpenKeyTable().get(multipartKey);
+ OmMultipartKeyInfo omMultipartKeyInfo =
+ metadataMgr.getMultipartInfoTable().get(multipartKey);
+
+ Assert.assertNotNull(omKeyInfo);
+ Assert.assertNotNull(omMultipartKeyInfo);
+ Assert.assertEquals(OzoneFSUtils.getFileName(keyName),
+ omKeyInfo.getKeyName());
+ Assert.assertEquals(uploadID, omMultipartKeyInfo.getUploadID());
+
+ long parentID = getParentID(volumeName, bucketName, keyName,
+ metadataMgr);
+
+ TreeMap<Integer, OzoneManagerProtocolProtos.PartKeyInfo> partKeyInfoMap =
+ omMultipartKeyInfo.getPartKeyInfoMap();
+ for (Map.Entry<Integer, OzoneManagerProtocolProtos.PartKeyInfo> entry :
+ partKeyInfoMap.entrySet()) {
+ OzoneManagerProtocolProtos.PartKeyInfo partKeyInfo = entry.getValue();
+ OmKeyInfo currentKeyPartInfo =
+ OmKeyInfo.getFromProtobuf(partKeyInfo.getPartKeyInfo());
+
+ Assert.assertEquals(OzoneFSUtils.getFileName(keyName),
+ currentKeyPartInfo.getKeyName());
+
+ // prepare dbPartName <parentID>/partFileName
+ String partFileName = OzoneFSUtils.getFileName(partName);
+ String dbPartName = metadataMgr.getOzonePathKey(parentID, partFileName);
+
+ Assert.assertEquals(dbPartName, partKeyInfo.getPartName());
+ }
+ return multipartKey;
+ }
+
+ private String getMultipartKey(String multipartUploadID, String volumeName,
+ String bucketName, String keyName, OMMetadataManager omMetadataManager)
+ throws IOException {
+
+ String fileName = OzoneFSUtils.getFileName(keyName);
+ long parentID = getParentID(volumeName, bucketName, keyName,
+ omMetadataManager);
+
+ String multipartKey = omMetadataManager.getMultipartKey(parentID,
+ fileName, multipartUploadID);
+
+ return multipartKey;
+ }
+
+ private long getParentID(String volumeName, String bucketName,
+ String keyName, OMMetadataManager omMetadataManager) throws IOException {
+ Iterator<Path> pathComponents = Paths.get(keyName).iterator();
+ String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
+ OmBucketInfo omBucketInfo =
+ omMetadataManager.getBucketTable().get(bucketKey);
+ long bucketId = omBucketInfo.getObjectID();
+ return OMFileRequest.getParentID(bucketId, pathComponents,
+ keyName, omMetadataManager);
+ }
+
private String initiateMultipartUpload(OzoneBucket bucket, String keyName,
ReplicationType replicationType, ReplicationFactor replicationFactor)
throws Exception {
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
index db9f32e..52305253 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.ozone.om.request.key.acl.prefix.OMPrefixSetAclRequest;
import org.apache.hadoop.ozone.om.request.s3.multipart.S3InitiateMultipartUploadRequest;
import org.apache.hadoop.ozone.om.request.s3.multipart.S3InitiateMultipartUploadRequestV1;
import org.apache.hadoop.ozone.om.request.s3.multipart.S3MultipartUploadAbortRequest;
+import org.apache.hadoop.ozone.om.request.s3.multipart.S3MultipartUploadAbortRequestV1;
import org.apache.hadoop.ozone.om.request.s3.multipart.S3MultipartUploadCommitPartRequest;
import org.apache.hadoop.ozone.om.request.s3.multipart.S3MultipartUploadCommitPartRequestV1;
import org.apache.hadoop.ozone.om.request.s3.multipart.S3MultipartUploadCompleteRequest;
@@ -191,6 +192,9 @@ public final class OzoneManagerRatisUtils {
}
return new S3MultipartUploadCommitPartRequest(omRequest);
case AbortMultiPartUpload:
+ if (isBucketFSOptimized()) {
+ return new S3MultipartUploadAbortRequestV1(omRequest);
+ }
return new S3MultipartUploadAbortRequest(omRequest);
case CompleteMultiPartUpload:
if (isBucketFSOptimized()) {
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java
index 650133b..fb67a20 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java
@@ -123,8 +123,8 @@ public class S3MultipartUploadAbortRequest extends OMKeyRequest {
validateBucketAndVolume(omMetadataManager, volumeName, bucketName);
- multipartKey = omMetadataManager.getMultipartKey(
- volumeName, bucketName, keyName, keyArgs.getMultipartUploadID());
+ multipartKey = getMultipartKey(keyArgs.getMultipartUploadID(),
+ volumeName, bucketName, keyName, omMetadataManager);
OmKeyInfo omKeyInfo =
omMetadataManager.getOpenKeyTable().get(multipartKey);
@@ -166,19 +166,14 @@ public class S3MultipartUploadAbortRequest extends OMKeyRequest {
new CacheKey<>(multipartKey),
new CacheValue<>(Optional.absent(), trxnLogIndex));
- omClientResponse = new S3MultipartUploadAbortResponse(
- omResponse.setAbortMultiPartUploadResponse(
- MultipartUploadAbortResponse.newBuilder()).build(),
- multipartKey, multipartKeyInfo, ozoneManager.isRatisEnabled(),
- omBucketInfo.copyObject());
+ omClientResponse = getOmClientResponse(ozoneManager, multipartKeyInfo,
+ multipartKey, omResponse, omBucketInfo);
result = Result.SUCCESS;
} catch (IOException ex) {
result = Result.FAILURE;
exception = ex;
- omClientResponse =
- new S3MultipartUploadAbortResponse(createErrorOMResponse(omResponse,
- exception));
+ omClientResponse = getOmClientResponse(exception, omResponse);
} finally {
addResponseToDoubleBuffer(trxnLogIndex, omClientResponse,
omDoubleBufferHelper);
@@ -212,4 +207,32 @@ public class S3MultipartUploadAbortRequest extends OMKeyRequest {
return omClientResponse;
}
+
+ protected OMClientResponse getOmClientResponse(IOException exception,
+ OMResponse.Builder omResponse) {
+
+ return new S3MultipartUploadAbortResponse(createErrorOMResponse(omResponse,
+ exception));
+ }
+
+ protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager,
+ OmMultipartKeyInfo multipartKeyInfo, String multipartKey,
+ OMResponse.Builder omResponse, OmBucketInfo omBucketInfo) {
+
+ OMClientResponse omClientResponse = new S3MultipartUploadAbortResponse(
+ omResponse.setAbortMultiPartUploadResponse(
+ MultipartUploadAbortResponse.newBuilder()).build(),
+ multipartKey, multipartKeyInfo, ozoneManager.isRatisEnabled(),
+ omBucketInfo.copyObject());
+ return omClientResponse;
+ }
+
+ protected String getMultipartKey(String multipartUploadID, String volumeName,
+ String bucketName, String keyName, OMMetadataManager omMetadataManager)
+ throws IOException {
+
+ String multipartKey = omMetadataManager.getMultipartKey(
+ volumeName, bucketName, keyName, multipartUploadID);
+ return multipartKey;
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequestV1.java
new file mode 100644
index 0000000..ddeb10c
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequestV1.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.request.s3.multipart;
+
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
+import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
+import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.om.response.s3.multipart.S3MultipartUploadAbortResponseV1;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartUploadAbortResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Iterator;
+
+/**
+ * Handles Abort of multipart upload request.
+ */
+public class S3MultipartUploadAbortRequestV1
+ extends S3MultipartUploadAbortRequest {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(S3MultipartUploadAbortRequestV1.class);
+
+ public S3MultipartUploadAbortRequestV1(OMRequest omRequest) {
+ super(omRequest);
+ }
+
+ protected OMClientResponse getOmClientResponse(IOException exception,
+ OMResponse.Builder omResponse) {
+
+ return new S3MultipartUploadAbortResponseV1(createErrorOMResponse(
+ omResponse, exception));
+ }
+
+ protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager,
+ OmMultipartKeyInfo multipartKeyInfo, String multipartKey,
+ OMResponse.Builder omResponse, OmBucketInfo omBucketInfo) {
+
+ OMClientResponse omClientResponse = new S3MultipartUploadAbortResponseV1(
+ omResponse.setAbortMultiPartUploadResponse(
+ MultipartUploadAbortResponse.newBuilder()).build(),
+ multipartKey, multipartKeyInfo, ozoneManager.isRatisEnabled(),
+ omBucketInfo.copyObject());
+ return omClientResponse;
+ }
+
+ protected String getMultipartKey(String multipartUploadID, String volumeName,
+ String bucketName, String keyName, OMMetadataManager omMetadataManager)
+ throws IOException {
+
+ String fileName = OzoneFSUtils.getFileName(keyName);
+ Iterator<Path> pathComponents = Paths.get(keyName).iterator();
+ String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
+ OmBucketInfo omBucketInfo =
+ omMetadataManager.getBucketTable().get(bucketKey);
+ long bucketId = omBucketInfo.getObjectID();
+ long parentID = OMFileRequest.getParentID(bucketId, pathComponents,
+ keyName, omMetadataManager);
+
+ String multipartKey = omMetadataManager.getMultipartKey(parentID,
+ fileName, multipartUploadID);
+
+ return multipartKey;
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequestV1.java
index 7aa21cf..0f23745 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequestV1.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequestV1.java
@@ -92,7 +92,7 @@ public class S3MultipartUploadCommitPartRequestV1
getOmRequest());
OMClientResponse omClientResponse = null;
OzoneManagerProtocolProtos.PartKeyInfo oldPartKeyInfo = null;
- String openKey = null;
+ String openFileKey = null;
OmKeyInfo omKeyInfo = null;
String multipartKey = null;
OmMultipartKeyInfo multipartKeyInfo = null;
@@ -127,14 +127,15 @@ public class S3MultipartUploadCommitPartRequestV1
long clientID = multipartCommitUploadPartRequest.getClientID();
- openKey = omMetadataManager.getOpenFileName(parentID, fileName, clientID);
+ openFileKey = omMetadataManager.getOpenFileName(parentID, fileName,
+ clientID);
omKeyInfo = OMFileRequest.getOmKeyInfoFromFileTable(true,
- omMetadataManager, openKey, keyName);
+ omMetadataManager, openFileKey, keyName);
if (omKeyInfo == null) {
throw new OMException("Failed to commit Multipart Upload key, as " +
- openKey + " entry is not found in the openKey table",
+ openFileKey + " entry is not found in the openFileTable",
KEY_NOT_FOUND);
}
@@ -150,7 +151,7 @@ public class S3MultipartUploadCommitPartRequestV1
/**
* Format of PartName stored into MultipartInfoTable is,
- * "fileName + ClientID".
+ * "<parentID>/fileName + ClientID".
*
* Contract is that all part names present in a multipart info will
* have same key prefix path.
@@ -159,7 +160,9 @@ public class S3MultipartUploadCommitPartRequestV1
* /vol1/buck1/a/b/c/part-1, /vol1/buck1/a/b/c/part-2,
* /vol1/buck1/a/b/c/part-n
*/
- dbPartName = fileName + clientID;
+ String ozoneFileKey = omMetadataManager.getOzonePathKey(parentID,
+ fileName);
+ dbPartName = ozoneFileKey + clientID;
if (multipartKeyInfo == null) {
// This can occur when user started uploading part by the time commit
@@ -204,7 +207,7 @@ public class S3MultipartUploadCommitPartRequestV1
trxnLogIndex));
omMetadataManager.getOpenKeyTable().addCacheEntry(
- new CacheKey<>(openKey),
+ new CacheKey<>(openFileKey),
new CacheValue<>(Optional.absent(), trxnLogIndex));
long scmBlockSize = ozoneManager.getScmBlockSize();
@@ -228,7 +231,7 @@ public class S3MultipartUploadCommitPartRequestV1
.setPartName(fullKeyPartName));
omClientResponse = new S3MultipartUploadCommitPartResponseV1(
- omResponse.build(), multipartKey, openKey,
+ omResponse.build(), multipartKey, openFileKey,
multipartKeyInfo, oldPartKeyInfo, omKeyInfo,
ozoneManager.isRatisEnabled(),
omBucketInfo.copyObject());
@@ -238,8 +241,8 @@ public class S3MultipartUploadCommitPartRequestV1
result = Result.FAILURE;
exception = ex;
omClientResponse = new S3MultipartUploadCommitPartResponseV1(
- createErrorOMResponse(omResponse, exception), multipartKey, openKey,
- multipartKeyInfo, oldPartKeyInfo, omKeyInfo,
+ createErrorOMResponse(omResponse, exception), multipartKey,
+ openFileKey, multipartKeyInfo, oldPartKeyInfo, omKeyInfo,
ozoneManager.isRatisEnabled(), copyBucketInfo);
} finally {
addResponseToDoubleBuffer(trxnLogIndex, omClientResponse,
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
index d396e8e..162cf2f 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
@@ -34,9 +34,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
-import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
-import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils;
import org.apache.hadoop.ozone.om.request.key.OMKeyRequest;
import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
@@ -433,22 +431,11 @@ public class S3MultipartUploadCompleteRequest extends OMKeyRequest {
return dataSize;
}
- private String preparePartName(String requestedVolume,
+ protected String preparePartName(String requestedVolume,
String requestedBucket, String keyName, PartKeyInfo partKeyInfo,
OMMetadataManager omMetadataManager) {
- String partName;
- if (OzoneManagerRatisUtils.isBucketFSOptimized()) {
- String parentPath = OzoneFSUtils.getParent(keyName);
- StringBuffer keyPath = new StringBuffer(parentPath);
- keyPath.append(partKeyInfo.getPartName());
-
- partName = omMetadataManager.getOzoneKey(requestedVolume,
- requestedBucket, keyPath.toString());
- } else {
- partName = partKeyInfo.getPartName();
- }
- return partName;
+ return partKeyInfo.getPartName();
}
private static String failureMessage(String volume, String bucket,
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestV1.java
index 4ab9ee7..37f2dad 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestV1.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestV1.java
@@ -252,6 +252,7 @@ public class S3MultipartUploadCompleteRequestV1
new CacheValue<>(Optional.absent(), transactionLogIndex));
}
+ @Override
protected void updatePrefixFSOInfo(OmKeyInfo dbOpenKeyInfo,
OmKeyInfo.Builder builder) {
// updates parentID and fileName
@@ -259,6 +260,21 @@ public class S3MultipartUploadCompleteRequestV1
builder.setFileName(dbOpenKeyInfo.getFileName());
}
+ @Override
+ protected String preparePartName(String requestedVolume,
+ String requestedBucket, String keyName, PartKeyInfo partKeyInfo,
+ OMMetadataManager omMetadataManager) {
+
+ String parentPath = OzoneFSUtils.getParent(keyName);
+ StringBuffer keyPath = new StringBuffer(parentPath);
+ String partFileName = OzoneFSUtils.getFileName(partKeyInfo.getPartName());
+ keyPath.append(partFileName);
+
+ return omMetadataManager.getOzoneKey(requestedVolume,
+ requestedBucket, keyPath.toString());
+ }
+
+
private static String failureMessage(String volume, String bucket,
String keyName) {
return "Complete Multipart Upload Failed: volume: " +
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadAbortResponseV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadAbortResponseV1.java
new file mode 100644
index 0000000..3ed05ef
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadAbortResponseV1.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.response.s3.multipart;
+
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
+import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+
+import javax.annotation.Nonnull;
+
+import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE;
+import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.MULTIPARTFILEINFO_TABLE;
+import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.OPEN_FILE_TABLE;
+
+/**
+ * Response for Multipart Abort Request layout version V1.
+ */
+@CleanupTableInfo(cleanupTables = {OPEN_FILE_TABLE, DELETED_TABLE,
+ MULTIPARTFILEINFO_TABLE})
+public class S3MultipartUploadAbortResponseV1
+ extends S3MultipartUploadAbortResponse {
+
+ public S3MultipartUploadAbortResponseV1(@Nonnull OMResponse omResponse,
+ String multipartKey, @Nonnull OmMultipartKeyInfo omMultipartKeyInfo,
+ boolean isRatisEnabled, @Nonnull OmBucketInfo omBucketInfo) {
+
+ super(omResponse, multipartKey, omMultipartKeyInfo, isRatisEnabled,
+ omBucketInfo);
+ }
+
+ /**
+ * For when the request is not successful.
+ * For a successful request, the other constructor should be used.
+ */
+ public S3MultipartUploadAbortResponseV1(@Nonnull OMResponse omResponse) {
+ super(omResponse);
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequest.java
index 088b232..01561e9 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequest.java
@@ -56,7 +56,7 @@ public class TestS3InitiateMultipartUploadRequest
bucketName, keyName);
S3InitiateMultipartUploadRequest s3InitiateMultipartUploadRequest =
- new S3InitiateMultipartUploadRequest(modifiedRequest);
+ getS3InitiateMultipartUploadReq(modifiedRequest);
OMClientResponse omClientResponse =
s3InitiateMultipartUploadRequest.validateAndUpdateCache(ozoneManager,
@@ -65,8 +65,8 @@ public class TestS3InitiateMultipartUploadRequest
Assert.assertEquals(OzoneManagerProtocolProtos.Status.OK,
omClientResponse.getOMResponse().getStatus());
- String multipartKey = omMetadataManager.getMultipartKey(volumeName,
- bucketName, keyName, modifiedRequest.getInitiateMultiPartUploadRequest()
+ String multipartKey = getMultipartKey(volumeName, bucketName, keyName,
+ modifiedRequest.getInitiateMultiPartUploadRequest()
.getKeyArgs().getMultipartUploadID());
Assert.assertNotNull(omMetadataManager.getOpenKeyTable().get(multipartKey));
@@ -102,7 +102,7 @@ public class TestS3InitiateMultipartUploadRequest
volumeName, bucketName, keyName);
S3InitiateMultipartUploadRequest s3InitiateMultipartUploadRequest =
- new S3InitiateMultipartUploadRequest(modifiedRequest);
+ getS3InitiateMultipartUploadReq(modifiedRequest);
OMClientResponse omClientResponse =
s3InitiateMultipartUploadRequest.validateAndUpdateCache(ozoneManager,
@@ -111,8 +111,8 @@ public class TestS3InitiateMultipartUploadRequest
Assert.assertEquals(OzoneManagerProtocolProtos.Status.BUCKET_NOT_FOUND,
omClientResponse.getOMResponse().getStatus());
- String multipartKey = omMetadataManager.getMultipartKey(volumeName,
- bucketName, keyName, modifiedRequest.getInitiateMultiPartUploadRequest()
+ String multipartKey = getMultipartKey(volumeName, bucketName, keyName,
+ modifiedRequest.getInitiateMultiPartUploadRequest()
.getKeyArgs().getMultipartUploadID());
Assert.assertNull(omMetadataManager.getOpenKeyTable().get(multipartKey));
@@ -130,7 +130,7 @@ public class TestS3InitiateMultipartUploadRequest
keyName);
S3InitiateMultipartUploadRequest s3InitiateMultipartUploadRequest =
- new S3InitiateMultipartUploadRequest(modifiedRequest);
+ getS3InitiateMultipartUploadReq(modifiedRequest);
OMClientResponse omClientResponse =
s3InitiateMultipartUploadRequest.validateAndUpdateCache(ozoneManager,
@@ -139,12 +139,18 @@ public class TestS3InitiateMultipartUploadRequest
Assert.assertEquals(OzoneManagerProtocolProtos.Status.VOLUME_NOT_FOUND,
omClientResponse.getOMResponse().getStatus());
- String multipartKey = omMetadataManager.getMultipartKey(volumeName,
- bucketName, keyName, modifiedRequest.getInitiateMultiPartUploadRequest()
+ String multipartKey = getMultipartKey(volumeName, bucketName, keyName,
+ modifiedRequest.getInitiateMultiPartUploadRequest()
.getKeyArgs().getMultipartUploadID());
Assert.assertNull(omMetadataManager.getOpenKeyTable().get(multipartKey));
Assert.assertNull(omMetadataManager.getMultipartInfoTable()
.get(multipartKey));
}
+
+ protected String getMultipartKey(String volumeName, String bucketName,
+ String keyName, String multipartUploadID) {
+ return omMetadataManager.getMultipartKey(volumeName,
+ bucketName, keyName, multipartUploadID);
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequestV1.java
index 5fa75ba..4c9dbb5 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequestV1.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequestV1.java
@@ -65,8 +65,8 @@ public class TestS3InitiateMultipartUploadRequestV1
OMRequest modifiedRequest = doPreExecuteInitiateMPUV1(volumeName,
bucketName, keyName);
- S3InitiateMultipartUploadRequestV1 s3InitiateMultipartUploadRequestV1 =
- new S3InitiateMultipartUploadRequestV1(modifiedRequest);
+ S3InitiateMultipartUploadRequest s3InitiateMultipartUploadRequestV1 =
+ getS3InitiateMultipartUploadReq(modifiedRequest);
OMClientResponse omClientResponse =
s3InitiateMultipartUploadRequestV1.validateAndUpdateCache(
@@ -112,54 +112,6 @@ public class TestS3InitiateMultipartUploadRequestV1
.getCreationTime());
}
- @Test
- public void testValidateAndUpdateCacheWithBucketNotFound() throws Exception {
- String volumeName = UUID.randomUUID().toString();
- String bucketName = UUID.randomUUID().toString();
- String keyName = UUID.randomUUID().toString();
-
- TestOMRequestUtils.addVolumeToDB(volumeName, omMetadataManager);
-
- OMRequest modifiedRequest = doPreExecuteInitiateMPU(
- volumeName, bucketName, keyName);
-
- S3InitiateMultipartUploadRequest s3InitiateMultipartUploadRequest =
- new S3InitiateMultipartUploadRequest(modifiedRequest);
-
- OMClientResponse omClientResponse =
- s3InitiateMultipartUploadRequest.validateAndUpdateCache(ozoneManager,
- 100L, ozoneManagerDoubleBufferHelper);
-
- Assert.assertEquals(OzoneManagerProtocolProtos.Status.BUCKET_NOT_FOUND,
- omClientResponse.getOMResponse().getStatus());
-
- Assert.assertTrue(omMetadataManager.getOpenKeyTable().isEmpty());
- Assert.assertTrue(omMetadataManager.getMultipartInfoTable().isEmpty());
- }
-
- @Test
- public void testValidateAndUpdateCacheWithVolumeNotFound() throws Exception {
- String volumeName = UUID.randomUUID().toString();
- String bucketName = UUID.randomUUID().toString();
- String keyName = UUID.randomUUID().toString();
-
- OMRequest modifiedRequest = doPreExecuteInitiateMPU(volumeName, bucketName,
- keyName);
-
- S3InitiateMultipartUploadRequest s3InitiateMultipartUploadRequest =
- new S3InitiateMultipartUploadRequest(modifiedRequest);
-
- OMClientResponse omClientResponse =
- s3InitiateMultipartUploadRequest.validateAndUpdateCache(ozoneManager,
- 100L, ozoneManagerDoubleBufferHelper);
-
- Assert.assertEquals(OzoneManagerProtocolProtos.Status.VOLUME_NOT_FOUND,
- omClientResponse.getOMResponse().getStatus());
-
- Assert.assertTrue(omMetadataManager.getOpenKeyTable().isEmpty());
- Assert.assertTrue(omMetadataManager.getMultipartInfoTable().isEmpty());
- }
-
private long verifyDirectoriesInDB(List<String> dirs, long bucketID)
throws IOException {
// bucketID is the parent
@@ -179,4 +131,9 @@ public class TestS3InitiateMultipartUploadRequestV1
}
return parentID;
}
+
+ protected S3InitiateMultipartUploadRequest getS3InitiateMultipartUploadReq(
+ OMRequest initiateMPURequest) {
+ return new S3InitiateMultipartUploadRequestV1(initiateMPURequest);
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java
index 16cb4ae..d08b4ae 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java
@@ -183,7 +183,7 @@ public class TestS3MultipartRequest {
S3MultipartUploadAbortRequest s3MultipartUploadAbortRequest =
- new S3MultipartUploadAbortRequest(omRequest);
+ getS3MultipartUploadAbortReq(omRequest);
OMRequest modifiedRequest =
s3MultipartUploadAbortRequest.preExecute(ozoneManager);
@@ -262,4 +262,9 @@ public class TestS3MultipartRequest {
return new S3InitiateMultipartUploadRequest(initiateMPURequest);
}
+ protected S3MultipartUploadAbortRequest getS3MultipartUploadAbortReq(
+ OMRequest omRequest) {
+ return new S3MultipartUploadAbortRequest(omRequest);
+ }
+
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadAbortRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadAbortRequest.java
index d0b61c7..9bff636 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadAbortRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadAbortRequest.java
@@ -50,16 +50,18 @@ public class TestS3MultipartUploadAbortRequest extends TestS3MultipartRequest {
public void testValidateAndUpdateCache() throws Exception {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
- String keyName = UUID.randomUUID().toString();
+ String keyName = getKeyName();
TestOMRequestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
omMetadataManager);
+ createParentPath(volumeName, bucketName);
+
OMRequest initiateMPURequest = doPreExecuteInitiateMPU(volumeName,
bucketName, keyName);
S3InitiateMultipartUploadRequest s3InitiateMultipartUploadRequest =
- new S3InitiateMultipartUploadRequest(initiateMPURequest);
+ getS3InitiateMultipartUploadReq(initiateMPURequest);
OMClientResponse omClientResponse =
s3InitiateMultipartUploadRequest.validateAndUpdateCache(ozoneManager,
@@ -73,15 +75,15 @@ public class TestS3MultipartUploadAbortRequest extends TestS3MultipartRequest {
multipartUploadID);
S3MultipartUploadAbortRequest s3MultipartUploadAbortRequest =
- new S3MultipartUploadAbortRequest(abortMPURequest);
+ getS3MultipartUploadAbortReq(abortMPURequest);
omClientResponse =
s3MultipartUploadAbortRequest.validateAndUpdateCache(ozoneManager, 2L,
ozoneManagerDoubleBufferHelper);
- String multipartKey = omMetadataManager.getMultipartKey(volumeName,
- bucketName, keyName, multipartUploadID);
+ String multipartKey = getMultipartKey(volumeName, bucketName, keyName,
+ multipartUploadID);
// Check table and response.
Assert.assertEquals(OzoneManagerProtocolProtos.Status.OK,
@@ -108,7 +110,7 @@ public class TestS3MultipartUploadAbortRequest extends TestS3MultipartRequest {
multipartUploadID);
S3MultipartUploadAbortRequest s3MultipartUploadAbortRequest =
- new S3MultipartUploadAbortRequest(abortMPURequest);
+ getS3MultipartUploadAbortReq(abortMPURequest);
OMClientResponse omClientResponse =
s3MultipartUploadAbortRequest.validateAndUpdateCache(ozoneManager, 2L,
@@ -177,4 +179,19 @@ public class TestS3MultipartUploadAbortRequest extends TestS3MultipartRequest {
omClientResponse.getOMResponse().getStatus());
}
+
+ protected String getKeyName() {
+ return UUID.randomUUID().toString();
+ }
+
+ protected void createParentPath(String volumeName, String bucketName)
+ throws Exception {
+ // no parent hierarchy
+ }
+
+ protected String getMultipartKey(String volumeName, String bucketName,
+ String keyName, String multipartUploadID) {
+ return omMetadataManager.getMultipartKey(volumeName,
+ bucketName, keyName, multipartUploadID);
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadAbortRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadAbortRequestV1.java
new file mode 100644
index 0000000..fd8a158
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadAbortRequestV1.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.request.s3.multipart;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.ozone.om.request.TestOMRequestUtils;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+
+import java.util.UUID;
+
+/**
+ * Test Multipart upload abort request.
+ */
+public class TestS3MultipartUploadAbortRequestV1
+ extends TestS3MultipartUploadAbortRequest {
+
+ private String dirName = "a/b/c/";
+
+ private long parentID;
+
+ @Override
+ protected S3MultipartUploadAbortRequest getS3MultipartUploadAbortReq(
+ OMRequest omRequest) {
+ return new S3MultipartUploadAbortRequestV1(omRequest);
+ }
+
+ @Override
+ protected S3InitiateMultipartUploadRequest getS3InitiateMultipartUploadReq(
+ OMRequest initiateMPURequest) {
+ return new S3InitiateMultipartUploadRequestV1(initiateMPURequest);
+ }
+
+ @Override
+ protected String getKeyName() {
+ return dirName + UUID.randomUUID().toString();
+ }
+
+ @Override
+ protected void createParentPath(String volumeName, String bucketName)
+ throws Exception {
+ // Create parent dirs for the path
+ parentID = TestOMRequestUtils.addParentsToDirTable(volumeName, bucketName,
+ dirName, omMetadataManager);
+ }
+
+ @Override
+ protected String getMultipartKey(String volumeName, String bucketName,
+ String keyName, String multipartUploadID) {
+ String fileName = StringUtils.substringAfter(keyName, dirName);
+ return omMetadataManager.getMultipartKey(parentID, fileName,
+ multipartUploadID);
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java
index 6f4d6fa..c10ff77 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java
@@ -116,8 +116,8 @@ public class TestS3MultipartResponse {
.setKeyName(keyName)
.setMultipartUploadID(multipartUploadID)).build();
- return new S3InitiateMultipartUploadResponse(omResponse, multipartKeyInfo,
- omKeyInfo);
+ return getS3InitiateMultipartUploadResp(multipartKeyInfo, omKeyInfo,
+ omResponse);
}
public S3MultipartUploadAbortResponse createS3AbortMPUResponse(
@@ -130,11 +130,10 @@ public class TestS3MultipartResponse {
.setAbortMultiPartUploadResponse(
MultipartUploadAbortResponse.newBuilder().build()).build();
- return new S3MultipartUploadAbortResponse(omResponse, multipartKey,
- omMultipartKeyInfo, true, omBucketInfo);
+ return getS3MultipartUploadAbortResp(multipartKey,
+ omMultipartKeyInfo, omBucketInfo, omResponse);
}
-
public void addPart(int partNumber, PartKeyInfo partKeyInfo,
OmMultipartKeyInfo omMultipartKeyInfo) {
omMultipartKeyInfo.addPartKeyInfo(partNumber, partKeyInfo);
@@ -162,7 +161,7 @@ public class TestS3MultipartResponse {
int partNumber) {
return PartKeyInfo.newBuilder()
.setPartNumber(partNumber)
- .setPartName(omMetadataManager.getMultipartKey(parentID, fileName,
+ .setPartName(omMetadataManager.getOzonePathKey(parentID, fileName +
UUID.randomUUID().toString()))
.setPartKeyInfo(KeyInfo.newBuilder()
.setVolumeName(volumeName)
@@ -295,4 +294,18 @@ public class TestS3MultipartResponse {
return omMetadataManager.getMultipartKey(parentID, fileName,
multipartUploadID);
}
+
+ protected S3InitiateMultipartUploadResponse getS3InitiateMultipartUploadResp(
+ OmMultipartKeyInfo multipartKeyInfo, OmKeyInfo omKeyInfo,
+ OMResponse omResponse) {
+ return new S3InitiateMultipartUploadResponse(omResponse, multipartKeyInfo,
+ omKeyInfo);
+ }
+
+ protected S3MultipartUploadAbortResponse getS3MultipartUploadAbortResp(
+ String multipartKey, OmMultipartKeyInfo omMultipartKeyInfo,
+ OmBucketInfo omBucketInfo, OMResponse omResponse) {
+ return new S3MultipartUploadAbortResponse(omResponse, multipartKey,
+ omMultipartKeyInfo, true, omBucketInfo);
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadAbortResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadAbortResponse.java
index a11c4db..ae8650e 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadAbortResponse.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadAbortResponse.java
@@ -43,10 +43,10 @@ public class TestS3MultipartUploadAbortResponse
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
- String keyName = UUID.randomUUID().toString();
+ String keyName = getKeyName();
String multipartUploadID = UUID.randomUUID().toString();
- String multipartKey = omMetadataManager.getMultipartKey(volumeName,
- bucketName, keyName, multipartUploadID);
+ String multipartKey = getMultipartKey(volumeName, bucketName, keyName,
+ multipartUploadID);
OmBucketInfo omBucketInfo = OmBucketInfo.newBuilder()
.setVolumeName(volumeName).setBucketName(bucketName)
@@ -82,10 +82,10 @@ public class TestS3MultipartUploadAbortResponse
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
- String keyName = UUID.randomUUID().toString();
+ String keyName = getKeyName();
String multipartUploadID = UUID.randomUUID().toString();
- String multipartKey = omMetadataManager.getMultipartKey(volumeName,
- bucketName, keyName, multipartUploadID);
+ String multipartKey = getMultipartKey(volumeName, bucketName, keyName,
+ multipartUploadID);
OmBucketInfo omBucketInfo = OmBucketInfo.newBuilder()
.setVolumeName(volumeName).setBucketName(bucketName)
@@ -154,4 +154,14 @@ public class TestS3MultipartUploadAbortResponse
ro.getOmKeyInfoList().get(0));
}
+ protected String getKeyName() {
+ return UUID.randomUUID().toString();
+ }
+
+ protected String getMultipartKey(String volumeName, String bucketName,
+ String keyName, String multipartUploadID) {
+ return omMetadataManager.getMultipartKey(volumeName,
+ bucketName, keyName, multipartUploadID);
+ }
+
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadAbortResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadAbortResponseV1.java
new file mode 100644
index 0000000..115b277
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadAbortResponseV1.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.response.s3.multipart;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+
+import java.util.ArrayList;
+import java.util.UUID;
+
+/**
+ * Test multipart upload abort response.
+ */
+public class TestS3MultipartUploadAbortResponseV1
+ extends TestS3MultipartUploadAbortResponse {
+
+ private String dirName = "abort/b/c/";
+
+ private long parentID = 1027;
+
+ protected String getKeyName() {
+ return dirName + UUID.randomUUID().toString();
+ }
+
+ protected String getMultipartKey(String volumeName, String bucketName,
+ String keyName, String multipartUploadID) {
+ String fileName = StringUtils.substringAfter(keyName, dirName);
+ return omMetadataManager.getMultipartKey(parentID, fileName,
+ multipartUploadID);
+ }
+
+ protected S3InitiateMultipartUploadResponse getS3InitiateMultipartUploadResp(
+ OmMultipartKeyInfo multipartKeyInfo, OmKeyInfo omKeyInfo,
+ OzoneManagerProtocolProtos.OMResponse omResponse) {
+ return new S3InitiateMultipartUploadResponseV1(omResponse, multipartKeyInfo,
+ omKeyInfo, new ArrayList<>());
+ }
+
+ protected S3MultipartUploadAbortResponse getS3MultipartUploadAbortResp(
+ String multipartKey, OmMultipartKeyInfo omMultipartKeyInfo,
+ OmBucketInfo omBucketInfo,
+ OzoneManagerProtocolProtos.OMResponse omResponse) {
+ return new S3MultipartUploadAbortResponseV1(omResponse, multipartKey,
+ omMultipartKeyInfo, true, omBucketInfo);
+ }
+
+ @Override
+ public OzoneManagerProtocolProtos.PartKeyInfo createPartKeyInfo(
+ String volumeName, String bucketName, String keyName, int partNumber) {
+
+ String fileName = OzoneFSUtils.getFileName(keyName);
+ return createPartKeyInfoV1(volumeName, bucketName, parentID, fileName,
+ partNumber);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org