You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@ozone.apache.org by GitBox <gi...@apache.org> on 2021/05/12 04:20:28 UTC
[GitHub] [ozone] rakeshadr commented on a change in pull request #2228: HDDS-5201. [FSO] S3MultiPart: Use existing ozone key format for MPU Info in DB
rakeshadr commented on a change in pull request #2228:
URL: https://github.com/apache/ozone/pull/2228#discussion_r630713350
##########
File path: hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequestWithFSO.java
##########
@@ -54,19 +54,20 @@ protected OMClientResponse getOmClientResponse(IOException exception,
protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager,
OmMultipartKeyInfo multipartKeyInfo, String multipartKey,
- OMResponse.Builder omResponse, OmBucketInfo omBucketInfo) {
+ String multipartOpenKey, OMResponse.Builder omResponse,
+ OmBucketInfo omBucketInfo) {
OMClientResponse omClientResp = new S3MultipartUploadAbortResponseWithFSO(
omResponse.setAbortMultiPartUploadResponse(
- MultipartUploadAbortResponse.newBuilder()).build(),
- multipartKey, multipartKeyInfo, ozoneManager.isRatisEnabled(),
+ MultipartUploadAbortResponse.newBuilder()).build(), multipartKey,
+ multipartOpenKey, multipartKeyInfo, ozoneManager.isRatisEnabled(),
omBucketInfo.copyObject());
return omClientResp;
}
- protected String getMultipartKey(String multipartUploadID, String volumeName,
- String bucketName, String keyName, OMMetadataManager omMetadataManager)
- throws IOException {
+ protected String getMultipartOpenKey(String multipartUploadID,
Review comment:
Done!
##########
File path: hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequestWithFSO.java
##########
@@ -62,197 +46,41 @@ public S3MultipartUploadCommitPartRequestWithFSO(OMRequest omRequest) {
}
@Override
- @SuppressWarnings("methodlength")
- public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
- long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) {
- MultipartCommitUploadPartRequest multipartCommitUploadPartRequest =
- getOmRequest().getCommitMultiPartUploadRequest();
-
- KeyArgs keyArgs = multipartCommitUploadPartRequest.getKeyArgs();
- Map<String, String> auditMap = buildKeyArgsAuditMap(keyArgs);
-
- String volumeName = keyArgs.getVolumeName();
- String bucketName = keyArgs.getBucketName();
- String keyName = keyArgs.getKeyName();
-
- OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
- ozoneManager.getMetrics().incNumCommitMultipartUploadParts();
-
- boolean acquiredLock = false;
-
- IOException exception = null;
- String dbPartName;
- String fullKeyPartName = null;
- OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
- getOmRequest());
- OMClientResponse omClientResponse = null;
- OzoneManagerProtocolProtos.PartKeyInfo oldPartKeyInfo = null;
- String openFileKey = null;
- OmKeyInfo omKeyInfo = null;
- String multipartKey = null;
- OmMultipartKeyInfo multipartKeyInfo = null;
- Result result;
- OmBucketInfo omBucketInfo;
- OmBucketInfo copyBucketInfo = null;
- try {
- keyArgs = resolveBucketLink(ozoneManager, keyArgs, auditMap);
- volumeName = keyArgs.getVolumeName();
- bucketName = keyArgs.getBucketName();
-
- // TODO to support S3 ACL later.
- acquiredLock = omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK,
- volumeName, bucketName);
-
- validateBucketAndVolume(omMetadataManager, volumeName, bucketName);
-
- String fileName = OzoneFSUtils.getFileName(keyName);
- Iterator<Path> pathComponents = Paths.get(keyName).iterator();
- String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
- omBucketInfo = omMetadataManager.getBucketTable().get(bucketKey);
- long bucketId = omBucketInfo.getObjectID();
- long parentID = OMFileRequest.getParentID(bucketId, pathComponents,
- keyName, omMetadataManager);
-
- String uploadID = keyArgs.getMultipartUploadID();
- multipartKey = omMetadataManager.getMultipartKey(parentID,
- fileName, uploadID);
-
- multipartKeyInfo = omMetadataManager.getMultipartInfoTable()
- .get(multipartKey);
-
- long clientID = multipartCommitUploadPartRequest.getClientID();
-
- openFileKey = omMetadataManager.getOpenFileName(parentID, fileName,
- clientID);
-
- omKeyInfo = OMFileRequest.getOmKeyInfoFromFileTable(true,
- omMetadataManager, openFileKey, keyName);
-
- if (omKeyInfo == null) {
- throw new OMException("Failed to commit Multipart Upload key, as " +
- openFileKey + " entry is not found in the openFileTable",
- KEY_NOT_FOUND);
- }
-
- // set the data size and location info list
- omKeyInfo.setDataSize(keyArgs.getDataSize());
- omKeyInfo.updateLocationInfoList(keyArgs.getKeyLocationsList().stream()
- .map(OmKeyLocationInfo::getFromProtobuf)
- .collect(Collectors.toList()), true);
- // Set Modification time
- omKeyInfo.setModificationTime(keyArgs.getModificationTime());
- // Set the UpdateID to current transactionLogIndex
- omKeyInfo.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled());
-
- /**
- * Format of PartName stored into MultipartInfoTable is,
- * "<parentID>/fileName + ClientID".
- *
- * Contract is that all part names present in a multipart info will
- * have same key prefix path.
- *
- * For example:
- * /vol1/buck1/a/b/c/part-1, /vol1/buck1/a/b/c/part-2,
- * /vol1/buck1/a/b/c/part-n
- */
- String ozoneFileKey = omMetadataManager.getOzonePathKey(parentID,
- fileName);
- dbPartName = ozoneFileKey + clientID;
-
- if (multipartKeyInfo == null) {
- // This can occur when user started uploading part by the time commit
- // of that part happens, in between the user might have requested
- // abort multipart upload. If we just throw exception, then the data
- // will not be garbage collected, so move this part to delete table
- // and throw error
- // Move this part to delete table.
- throw new OMException("No such Multipart upload is with specified " +
- "uploadId " + uploadID,
- OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR);
- }
-
- int partNumber = keyArgs.getMultipartNumber();
- oldPartKeyInfo = multipartKeyInfo.getPartKeyInfo(partNumber);
-
- // Build this multipart upload part info.
- OzoneManagerProtocolProtos.PartKeyInfo.Builder partKeyInfo =
- OzoneManagerProtocolProtos.PartKeyInfo.newBuilder();
- partKeyInfo.setPartName(dbPartName);
- partKeyInfo.setPartNumber(partNumber);
- partKeyInfo.setPartKeyInfo(omKeyInfo.getProtobuf(fileName,
- getOmRequest().getVersion()));
-
- // Add this part information in to multipartKeyInfo.
- multipartKeyInfo.addPartKeyInfo(partNumber, partKeyInfo.build());
-
- // Set the UpdateID to current transactionLogIndex
- multipartKeyInfo.setUpdateID(trxnLogIndex,
- ozoneManager.isRatisEnabled());
-
- // OldPartKeyInfo will be deleted. Its updateID will be set in
- // S3MultipartUplodaCommitPartResponse before being added to
- // DeletedKeyTable.
-
- // Delete from open key table and add it to multipart info table.
- // No need to add cache entries to delete table, as no
- // read/write requests that info for validation.
- omMetadataManager.getMultipartInfoTable().addCacheEntry(
- new CacheKey<>(multipartKey),
- new CacheValue<>(Optional.of(multipartKeyInfo),
- trxnLogIndex));
-
- omMetadataManager.getOpenKeyTable().addCacheEntry(
- new CacheKey<>(openFileKey),
- new CacheValue<>(Optional.absent(), trxnLogIndex));
-
- long scmBlockSize = ozoneManager.getScmBlockSize();
- int factor = omKeyInfo.getFactor().getNumber();
- omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName);
- // Block was pre-requested and UsedBytes updated when createKey and
- // AllocatedBlock. The space occupied by the Key shall be based on
- // the actual Key size, and the total Block size applied before should
- // be subtracted.
- long correctedSpace = omKeyInfo.getDataSize() * factor -
- keyArgs.getKeyLocationsList().size() * scmBlockSize * factor;
- omBucketInfo.incrUsedBytes(correctedSpace);
-
- // Prepare response. Sets user given full key part name in 'partName'
- // attribute in response object.
- String fullOzoneKeyName = omMetadataManager.getOzoneKey(
- volumeName, bucketName, keyName);
- fullKeyPartName = fullOzoneKeyName + clientID;
- omResponse.setCommitMultiPartUploadResponse(
- MultipartCommitUploadPartResponse.newBuilder()
- .setPartName(fullKeyPartName));
-
- omClientResponse = new S3MultipartUploadCommitPartResponseWithFSO(
- omResponse.build(), multipartKey, openFileKey,
- multipartKeyInfo, oldPartKeyInfo, omKeyInfo,
- ozoneManager.isRatisEnabled(),
- omBucketInfo.copyObject());
+ protected String getOpenKey(String volumeName, String bucketName,
+ String keyName, OMMetadataManager omMetadataManager, long clientID)
+ throws IOException {
+
+ String fileName = OzoneFSUtils.getFileName(keyName);
+ Iterator<Path> pathComponents = Paths.get(keyName).iterator();
+ String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
+ OmBucketInfo omBucketInfo =
+ omMetadataManager.getBucketTable().get(bucketKey);
+ long bucketId = omBucketInfo.getObjectID();
+ long parentID = OMFileRequest
+ .getParentID(bucketId, pathComponents, keyName, omMetadataManager);
+
+ return omMetadataManager.getOpenFileName(parentID, fileName, clientID);
+ }
- result = Result.SUCCESS;
- } catch (IOException ex) {
- result = Result.FAILURE;
- exception = ex;
- omClientResponse = new S3MultipartUploadCommitPartResponseWithFSO(
- createErrorOMResponse(omResponse, exception), multipartKey,
- openFileKey, multipartKeyInfo, oldPartKeyInfo, omKeyInfo,
- ozoneManager.isRatisEnabled(), copyBucketInfo);
- } finally {
- addResponseToDoubleBuffer(trxnLogIndex, omClientResponse,
- omDoubleBufferHelper);
- if (acquiredLock) {
- omMetadataManager.getLock().releaseWriteLock(BUCKET_LOCK,
- volumeName, bucketName);
- }
- }
+ @Override
+ protected OmKeyInfo getOmKeyInfo(OMMetadataManager omMetadataManager,
+ String openKey, String keyName) throws IOException {
- logResult(ozoneManager, multipartCommitUploadPartRequest, keyArgs,
- auditMap, volumeName, bucketName, keyName, exception,
- fullKeyPartName, result);
+ return OMFileRequest.getOmKeyInfoFromFileTable(true,
+ omMetadataManager, openKey, keyName);
+ }
- return omClientResponse;
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ protected S3MultipartUploadCommitPartResponse getOmClientResponse(
Review comment:
Done!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org