You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ar...@apache.org on 2020/07/24 15:58:15 UTC
[hadoop-ozone] branch master updated: HDDS-3999. OM Shutdown when
Commit part tries to commit the part, after abort upload. (#1244)
This is an automated email from the ASF dual-hosted git repository.
arp pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 32ac7bf HDDS-3999. OM Shutdown when Commit part tries to commit the part, after abort upload. (#1244)
32ac7bf is described below
commit 32ac7bf79529e30f2ec170456bf3a8803a380edd
Author: Bharat Viswanadham <bh...@apache.org>
AuthorDate: Fri Jul 24 08:57:05 2020 -0700
HDDS-3999. OM Shutdown when Commit part tries to commit the part, after abort upload. (#1244)
---
.../client/rpc/TestOzoneRpcClientAbstract.java | 105 ++++++++++++++++++++-
.../S3MultipartUploadCommitPartResponse.java | 3 +-
2 files changed, 104 insertions(+), 4 deletions(-)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
index ac9faa6..32b6bca 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
@@ -25,6 +25,7 @@ import java.util.Arrays;
import java.util.BitSet;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -108,6 +109,7 @@ import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE;
import static org.apache.hadoop.hdds.client.ReplicationType.STAND_ALONE;
import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS;
import static org.apache.hadoop.ozone.OzoneAcl.AclScope.DEFAULT;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR;
import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType.GROUP;
import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType.USER;
import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.READ;
@@ -1752,7 +1754,7 @@ public abstract class TestOzoneRpcClientAbstract {
String uploadID = "random";
OzoneTestUtils
- .expectOmException(ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR, () ->
+ .expectOmException(NO_SUCH_MULTIPART_UPLOAD_ERROR, () ->
bucket
.createMultipartKey(keyName, sampleData.length(), 1, uploadID));
}
@@ -1911,10 +1913,107 @@ public abstract class TestOzoneRpcClientAbstract {
volume.createBucket(bucketName);
OzoneBucket bucket = volume.getBucket(bucketName);
- OzoneTestUtils.expectOmException(ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR,
+ OzoneTestUtils.expectOmException(NO_SUCH_MULTIPART_UPLOAD_ERROR,
() -> bucket.abortMultipartUpload(keyName, "random"));
}
+ @Test
+ public void testAbortUploadFailWithInProgressPartUpload() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String keyName = UUID.randomUUID().toString();
+
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ OmMultipartInfo omMultipartInfo = bucket.initiateMultipartUpload(keyName,
+ STAND_ALONE, ONE);
+
+ Assert.assertNotNull(omMultipartInfo.getUploadID());
+
+ // Do not close output stream.
+ byte[] data = "data".getBytes(UTF_8);
+ OzoneOutputStream ozoneOutputStream = bucket.createMultipartKey(keyName,
+ data.length, 1, omMultipartInfo.getUploadID());
+ ozoneOutputStream.write(data, 0, data.length);
+
+ // Abort before completing part upload.
+ bucket.abortMultipartUpload(keyName, omMultipartInfo.getUploadID());
+
+ try {
+ ozoneOutputStream.close();
+ fail("testAbortUploadFailWithInProgressPartUpload failed");
+ } catch (IOException ex) {
+ assertTrue(ex instanceof OMException);
+ assertEquals(NO_SUCH_MULTIPART_UPLOAD_ERROR,
+ ((OMException) ex).getResult());
+ }
+ }
+
+ @Test
+ public void testCommitPartAfterCompleteUpload() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String keyName = UUID.randomUUID().toString();
+
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ OmMultipartInfo omMultipartInfo = bucket.initiateMultipartUpload(keyName,
+ STAND_ALONE, ONE);
+
+ Assert.assertNotNull(omMultipartInfo.getUploadID());
+
+ String uploadID = omMultipartInfo.getUploadID();
+
+ // upload part 1.
+ byte[] data = generateData(5 * 1024 * 1024,
+ (byte) RandomUtils.nextLong());
+ OzoneOutputStream ozoneOutputStream = bucket.createMultipartKey(keyName,
+ data.length, 1, uploadID);
+ ozoneOutputStream.write(data, 0, data.length);
+ ozoneOutputStream.close();
+
+ OmMultipartCommitUploadPartInfo omMultipartCommitUploadPartInfo =
+ ozoneOutputStream.getCommitUploadPartInfo();
+
+ // Do not close output stream for part 2.
+ ozoneOutputStream = bucket.createMultipartKey(keyName,
+ data.length, 2, omMultipartInfo.getUploadID());
+ ozoneOutputStream.write(data, 0, data.length);
+
+ Map<Integer, String> partsMap = new LinkedHashMap<>();
+ partsMap.put(1, omMultipartCommitUploadPartInfo.getPartName());
+ OmMultipartUploadCompleteInfo omMultipartUploadCompleteInfo =
+ bucket.completeMultipartUpload(keyName,
+ uploadID, partsMap);
+
+ Assert.assertNotNull(omMultipartCommitUploadPartInfo);
+
+ byte[] fileContent = new byte[data.length];
+ OzoneInputStream inputStream = bucket.readKey(keyName);
+ inputStream.read(fileContent);
+ StringBuilder sb = new StringBuilder(data.length);
+
+ // Combine all parts data, and check is it matching with get key data.
+ String part1 = new String(data);
+ sb.append(part1);
+ Assert.assertEquals(sb.toString(), new String(fileContent));
+
+ try {
+ ozoneOutputStream.close();
+ fail("testCommitPartAfterCompleteUpload failed");
+ } catch (IOException ex) {
+ assertTrue(ex instanceof OMException);
+ assertEquals(NO_SUCH_MULTIPART_UPLOAD_ERROR,
+ ((OMException) ex).getResult());
+ }
+ }
+
@Test
public void testAbortUploadSuccessWithOutAnyParts() throws Exception {
@@ -2142,7 +2241,7 @@ public abstract class TestOzoneRpcClientAbstract {
@Test
public void testListPartsWithInvalidUploadID() throws Exception {
OzoneTestUtils
- .expectOmException(ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR, () -> {
+ .expectOmException(NO_SUCH_MULTIPART_UPLOAD_ERROR, () -> {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
String keyName = UUID.randomUUID().toString();
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCommitPartResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCommitPartResponse.java
index 28acdb5..f68af4a 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCommitPartResponse.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCommitPartResponse.java
@@ -96,7 +96,8 @@ public class S3MultipartUploadCommitPartResponse extends OMClientResponse {
repeatedOmKeyInfo =
OmUtils.prepareKeyForDelete(openPartKeyInfoToBeDeleted,
- repeatedOmKeyInfo, omMultipartKeyInfo.getUpdateID(), isRatisEnabled);
+ repeatedOmKeyInfo, openPartKeyInfoToBeDeleted.getUpdateID(),
+ isRatisEnabled);
omMetadataManager.getDeletedTable().putWithBatch(batchOperation,
openKey, repeatedOmKeyInfo);
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org