You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2023/02/18 16:44:39 UTC
[ozone] branch master updated: HDDS-7596. Fix OM crash due to a corner case for FSO-enabled bucket (#4048)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new b466cd582e HDDS-7596. Fix OM crash due to a corner case for FSO-enabled bucket (#4048)
b466cd582e is described below
commit b466cd582e1d64bf611e0bb1f8daf66b89d006e7
Author: k5342 <k5...@users.noreply.github.com>
AuthorDate: Sun Feb 19 01:44:32 2023 +0900
HDDS-7596. Fix OM crash due to a corner case for FSO-enabled bucket (#4048)
---
.../S3MultipartUploadCompleteRequest.java | 5 +-
.../S3MultipartUploadCompleteRequestWithFSO.java | 5 +-
.../S3MultipartUploadCompleteResponse.java | 15 +---
.../S3MultipartUploadCompleteResponseWithFSO.java | 10 ++-
.../s3/multipart/TestS3MultipartResponse.java | 2 +-
...stS3MultipartUploadCompleteResponseWithFSO.java | 85 ++++++++++++++++++++++
6 files changed, 101 insertions(+), 21 deletions(-)
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
index 1c748930e8..8b9f69bb6d 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
@@ -262,10 +262,11 @@ public class S3MultipartUploadCompleteRequest extends OMKeyRequest {
.setHash(DigestUtils.sha256Hex(keyName)));
long volumeId = omMetadataManager.getVolumeId(volumeName);
+ long bucketId = omMetadataManager.getBucketId(volumeName, bucketName);
omClientResponse =
getOmClientResponse(multipartKey, omResponse, dbMultipartOpenKey,
omKeyInfo, unUsedParts, omBucketInfo, oldKeyVersionsToDelete,
- volumeId);
+ volumeId, bucketId);
result = Result.SUCCESS;
} else {
@@ -305,7 +306,7 @@ public class S3MultipartUploadCompleteRequest extends OMKeyRequest {
OMResponse.Builder omResponse, String dbMultipartOpenKey,
OmKeyInfo omKeyInfo, List<OmKeyInfo> unUsedParts,
OmBucketInfo omBucketInfo, RepeatedOmKeyInfo oldKeyVersionsToDelete,
- long volumeId) {
+ long volumeId, long bucketId) {
return new S3MultipartUploadCompleteResponse(omResponse.build(),
multipartKey, dbMultipartOpenKey, omKeyInfo, unUsedParts,
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java
index b6beaafa20..ae31f949fc 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java
@@ -163,11 +163,12 @@ public class S3MultipartUploadCompleteRequestWithFSO
OzoneManagerProtocolProtos.OMResponse.Builder omResponse,
String dbMultipartOpenKey, OmKeyInfo omKeyInfo,
List<OmKeyInfo> unUsedParts, OmBucketInfo omBucketInfo,
- RepeatedOmKeyInfo oldKeyVersionsToDelete, long volumeId) {
+ RepeatedOmKeyInfo oldKeyVersionsToDelete, long volumeId, long bucketId) {
return new S3MultipartUploadCompleteResponseWithFSO(omResponse.build(),
multipartKey, dbMultipartOpenKey, omKeyInfo, unUsedParts,
- getBucketLayout(), omBucketInfo, oldKeyVersionsToDelete, volumeId);
+ getBucketLayout(), omBucketInfo, oldKeyVersionsToDelete,
+ volumeId, bucketId);
}
@Override
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponse.java
index a3e9f0d5c5..52448b0a95 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponse.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponse.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.OMResponse;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
+import javax.annotation.CheckForNull;
import javax.annotation.Nonnull;
import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.BUCKET_TABLE;
@@ -66,7 +67,7 @@ public class S3MultipartUploadCompleteResponse extends OmKeyResponse {
@Nonnull OmKeyInfo omKeyInfo,
@Nonnull List<OmKeyInfo> unUsedParts,
@Nonnull BucketLayout bucketLayout,
- @Nonnull OmBucketInfo omBucketInfo,
+ @CheckForNull OmBucketInfo omBucketInfo,
RepeatedOmKeyInfo keyVersionsToDelete) {
super(omResponse, bucketLayout);
this.partsUnusedList = unUsedParts;
@@ -135,19 +136,7 @@ public class S3MultipartUploadCompleteResponse extends OmKeyResponse {
return ozoneKey;
}
- protected String getMultipartKey() {
- return multipartKey;
- }
-
protected OmKeyInfo getOmKeyInfo() {
return omKeyInfo;
}
-
- protected List<OmKeyInfo> getPartsUnusedList() {
- return partsUnusedList;
- }
-
- public OmBucketInfo getOmBucketInfo() {
- return omBucketInfo;
- }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java
index 8eec71ded3..1b3389d7cf 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+import javax.annotation.CheckForNull;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.List;
@@ -51,6 +52,7 @@ public class S3MultipartUploadCompleteResponseWithFSO
extends S3MultipartUploadCompleteResponse {
private long volumeId;
+ private long bucketId;
@SuppressWarnings("checkstyle:ParameterNumber")
public S3MultipartUploadCompleteResponseWithFSO(
@@ -60,11 +62,13 @@ public class S3MultipartUploadCompleteResponseWithFSO
@Nonnull OmKeyInfo omKeyInfo,
@Nonnull List<OmKeyInfo> unUsedParts,
@Nonnull BucketLayout bucketLayout,
- @Nonnull OmBucketInfo omBucketInfo,
- RepeatedOmKeyInfo keysToDelete, @Nonnull long volumeId) {
+ @CheckForNull OmBucketInfo omBucketInfo,
+ RepeatedOmKeyInfo keysToDelete,
+ @Nonnull long volumeId, @Nonnull long bucketId) {
super(omResponse, multipartKey, multipartOpenKey, omKeyInfo, unUsedParts,
bucketLayout, omBucketInfo, keysToDelete);
this.volumeId = volumeId;
+ this.bucketId = bucketId;
}
/**
@@ -87,7 +91,7 @@ public class S3MultipartUploadCompleteResponseWithFSO
OMFileRequest
.addToFileTable(omMetadataManager, batchOperation, getOmKeyInfo(),
- volumeId, getOmBucketInfo().getObjectID());
+ volumeId, bucketId);
return ozoneKey;
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java
index 378f102053..ca5db71006 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java
@@ -325,7 +325,7 @@ public class TestS3MultipartResponse {
return new S3MultipartUploadCompleteResponseWithFSO(omResponse,
multipartKey, multipartOpenKey, omKeyInfo, unUsedParts,
- getBucketLayout(), omBucketInfo, keysToDelete, volumeId);
+ getBucketLayout(), omBucketInfo, keysToDelete, volumeId, bucketId);
}
protected S3InitiateMultipartUploadResponse getS3InitiateMultipartUploadResp(
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadCompleteResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadCompleteResponseWithFSO.java
index 954aaea7b9..f09e6dcf70 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadCompleteResponseWithFSO.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadCompleteResponseWithFSO.java
@@ -133,6 +133,91 @@ public class TestS3MultipartUploadCompleteResponseWithFSO
omMetadataManager.getDeletedTable()));
}
+
+
+ @Test
+ // similar to testAddDBToBatch(), but omBucketInfo is null
+ public void testAddDBToBatchWithNullBucketInfo() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String keyName = getKeyName();
+ String multipartUploadID = UUID.randomUUID().toString();
+
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager);
+
+ long txnId = 150;
+ long objectId = parentID + 1;
+ String fileName = OzoneFSUtils.getFileName(keyName);
+ final long volumeId = omMetadataManager.getVolumeId(volumeName);
+ final long bucketId = omMetadataManager.getBucketId(volumeName,
+ bucketName);
+ String dbMultipartKey = omMetadataManager.getMultipartKey(volumeName,
+ bucketName, keyName, multipartUploadID);
+ String dbMultipartOpenKey = omMetadataManager.getMultipartKey(volumeId,
+ bucketId, parentID, fileName, multipartUploadID);
+ long clientId = Time.now();
+
+ // add MPU entry to OpenFileTable
+ List<OmDirectoryInfo> parentDirInfos = new ArrayList<>();
+ S3InitiateMultipartUploadResponse s3InitiateMultipartUploadResponseFSO =
+ createS3InitiateMPUResponseFSO(volumeName, bucketName, parentID,
+ keyName, multipartUploadID, parentDirInfos, volumeId, bucketId);
+
+ s3InitiateMultipartUploadResponseFSO.addToDBBatch(omMetadataManager,
+ batchOperation);
+
+ omMetadataManager.getStore().commitBatchOperation(batchOperation);
+
+ String dbOpenKey = omMetadataManager.getOpenFileName(volumeId, bucketId,
+ parentID, fileName, clientId);
+ String dbKey = omMetadataManager.getOzonePathKey(volumeId, bucketId,
+ parentID, fileName);
+ OmKeyInfo omKeyInfoFSO =
+ OMRequestTestUtils.createOmKeyInfo(volumeName, bucketName, keyName,
+ HddsProtos.ReplicationType.RATIS,
+ HddsProtos.ReplicationFactor.ONE, objectId, parentID, txnId,
+ Time.now());
+
+ // add key to openFileTable
+ omKeyInfoFSO.setKeyName(fileName);
+ OMRequestTestUtils.addFileToKeyTable(true, false,
+ fileName, omKeyInfoFSO, clientId, omKeyInfoFSO.getObjectID(),
+ omMetadataManager);
+
+ addS3MultipartUploadCommitPartResponseFSO(volumeName, bucketName, keyName,
+ multipartUploadID, dbOpenKey);
+
+ Assert.assertNotNull(
+ omMetadataManager.getMultipartInfoTable().get(dbMultipartKey));
+ Assert.assertNotNull(omMetadataManager.getOpenKeyTable(getBucketLayout())
+ .get(dbMultipartOpenKey));
+
+ // S3MultipartUploadCompleteResponseWithFSO should accept null bucketInfo
+ List<OmKeyInfo> unUsedParts = new ArrayList<>();
+ S3MultipartUploadCompleteResponse s3MultipartUploadCompleteResponse =
+ createS3CompleteMPUResponseFSO(volumeName, bucketName, parentID,
+ keyName, multipartUploadID, omKeyInfoFSO,
+ OzoneManagerProtocolProtos.Status.OK, unUsedParts,
+ null, null);
+
+ s3MultipartUploadCompleteResponse.addToDBBatch(omMetadataManager,
+ batchOperation);
+
+ omMetadataManager.getStore().commitBatchOperation(batchOperation);
+
+ Assert.assertNotNull(
+ omMetadataManager.getKeyTable(getBucketLayout()).get(dbKey));
+ Assert.assertNull(
+ omMetadataManager.getMultipartInfoTable().get(dbMultipartKey));
+ Assert.assertNull(omMetadataManager.getOpenKeyTable(getBucketLayout())
+ .get(dbMultipartOpenKey));
+
+ // As no parts are created, so no entries should be there in delete table.
+ Assert.assertEquals(0, omMetadataManager.countRowsInTable(
+ omMetadataManager.getDeletedTable()));
+ }
+
@Test
public void testAddDBToBatchWithParts() throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org