You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ca...@apache.org on 2022/05/10 11:28:37 UTC
[ozone] branch master updated: HDDS-6709. Fix bucket usedBytes while versioning is true. (#3388)
This is an automated email from the ASF dual-hosted git repository.
captainzmc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new ff351025bc HDDS-6709. Fix bucket usedBytes while versioning is true. (#3388)
ff351025bc is described below
commit ff351025bcd261863cbeaa5e223a319643581bd6
Author: micah zhao <ca...@apache.org>
AuthorDate: Tue May 10 19:28:32 2022 +0800
HDDS-6709. Fix bucket usedBytes while versioning is true. (#3388)
---
.../client/rpc/TestOzoneRpcClientAbstract.java | 36 ++++++++++++++++++++++
.../ozone/om/request/key/OMKeyCommitRequest.java | 12 +++-----
.../om/request/key/OMKeyCommitRequestWithFSO.java | 11 +++----
.../S3MultipartUploadCompleteRequest.java | 4 +--
4 files changed, 47 insertions(+), 16 deletions(-)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
index 5ae28dfb42..636535cd10 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
@@ -1039,6 +1039,42 @@ public abstract class TestOzoneRpcClientAbstract {
store.getVolume(volumeName).getBucket(bucketName).getUsedBytes());
}
+ static Stream<BucketLayout> bucketLayouts() {
+ return Stream.of(
+ BucketLayout.OBJECT_STORE,
+ BucketLayout.FILE_SYSTEM_OPTIMIZED
+ );
+ }
+
+ @ParameterizedTest
+ @MethodSource("bucketLayouts")
+ void bucketUsedBytesOverWrite(BucketLayout bucketLayout)
+ throws IOException {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ OzoneVolume volume = null;
+ String value = "sample value";
+ int valueLength = value.getBytes(UTF_8).length;
+ store.createVolume(volumeName);
+ volume = store.getVolume(volumeName);
+ BucketArgs bucketArgs = BucketArgs.newBuilder()
+ .setBucketLayout(bucketLayout).setVersioning(true).build();
+ volume.createBucket(bucketName, bucketArgs);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+ String keyName = UUID.randomUUID().toString();
+
+ writeKey(bucket, keyName, ONE, value, valueLength);
+ Assert.assertEquals(valueLength,
+ store.getVolume(volumeName).getBucket(bucketName).getUsedBytes());
+
+ // Overwrite the same key, because this bucket setVersioning is true,
+ // so the bucket usedBytes should increase.
+ writeKey(bucket, keyName, ONE, value, valueLength);
+ Assert.assertEquals(valueLength * 2,
+ store.getVolume(volumeName).getBucket(bucketName).getUsedBytes());
+ }
+
+
// TODO: testBucketQuota overlaps with testBucketUsedBytes,
// do cleanup when EC branch gets merged into master.
@ParameterizedTest
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
index 47e8cf32c2..6f9cafe4ec 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
@@ -203,11 +203,6 @@ public class OMKeyCommitRequest extends OMKeyRequest {
RepeatedOmKeyInfo oldKeyVersionsToDelete = null;
OmKeyInfo keyToDelete =
omMetadataManager.getKeyTable(getBucketLayout()).get(dbOzoneKey);
- if (keyToDelete != null && !omBucketInfo.getIsVersionEnabled()) {
- oldKeyVersionsToDelete = getOldVersionsToCleanUp(dbOzoneKey,
- keyToDelete, omMetadataManager,
- trxnLogIndex, ozoneManager.isRatisEnabled());
- }
omKeyInfo =
omMetadataManager.getOpenKeyTable(getBucketLayout()).get(dbOpenKey);
@@ -224,9 +219,12 @@ public class OMKeyCommitRequest extends OMKeyRequest {
omKeyInfo.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled());
long correctedSpace = omKeyInfo.getReplicatedSize();
- // Subtract the size of blocks to be overwritten.
- if (keyToDelete != null) {
+ if (keyToDelete != null && !omBucketInfo.getIsVersionEnabled()) {
+ // Subtract the size of blocks to be overwritten.
correctedSpace -= keyToDelete.getReplicatedSize();
+ oldKeyVersionsToDelete = getOldVersionsToCleanUp(dbOzoneKey,
+ keyToDelete, omMetadataManager,
+ trxnLogIndex, ozoneManager.isRatisEnabled());
} else {
// if keyToDelete isn't null, usedNamespace needn't check and
// increase.
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
index 54d1f7ab44..3e4d35ee65 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
@@ -155,16 +155,15 @@ public class OMKeyCommitRequestWithFSO extends OMKeyCommitRequest {
RepeatedOmKeyInfo oldKeyVersionsToDelete = null;
OmKeyInfo keyToDelete =
omMetadataManager.getKeyTable(getBucketLayout()).get(dbFileKey);
+
+ long correctedSpace = omKeyInfo.getReplicatedSize();
+
if (keyToDelete != null && !omBucketInfo.getIsVersionEnabled()) {
+ // Subtract the size of blocks to be overwritten.
+ correctedSpace -= keyToDelete.getReplicatedSize();
oldKeyVersionsToDelete = getOldVersionsToCleanUp(dbFileKey,
keyToDelete, omMetadataManager,
trxnLogIndex, ozoneManager.isRatisEnabled());
- }
-
- long correctedSpace = omKeyInfo.getReplicatedSize();
- // Subtract the size of blocks to be overwritten.
- if (keyToDelete != null) {
- correctedSpace -= keyToDelete.getReplicatedSize();
} else {
// if keyToDelete isn't null, usedNamespace shouldn't check and
// increase.
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
index 3f205a8cd8..d97220c7ce 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
@@ -222,13 +222,11 @@ public class S3MultipartUploadCompleteRequest extends OMKeyRequest {
RepeatedOmKeyInfo oldKeyVersionsToDelete = null;
OmKeyInfo keyToDelete =
omMetadataManager.getKeyTable(getBucketLayout()).get(dbOzoneKey);
+ long usedBytesDiff = 0;
if (keyToDelete != null && !omBucketInfo.getIsVersionEnabled()) {
oldKeyVersionsToDelete = getOldVersionsToCleanUp(dbOzoneKey,
keyToDelete, omMetadataManager,
trxnLogIndex, ozoneManager.isRatisEnabled());
- }
- long usedBytesDiff = 0;
- if (keyToDelete != null) {
long numCopy = keyToDelete.getReplicationConfig().getRequiredNodes();
usedBytesDiff -= keyToDelete.getDataSize() * numCopy;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org