You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ca...@apache.org on 2020/12/24 08:48:04 UTC
[ozone] branch master updated: HDDS-4277. Support Bucket Namespace
Quota Updates (#1706)
This is an automated email from the ASF dual-hosted git repository.
captainzmc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new a58d3f5 HDDS-4277. Support Bucket Namespace Quota Updates (#1706)
a58d3f5 is described below
commit a58d3f5a0046f836b30f246c3e98d51805addadd
Author: Rui Wang <am...@users.noreply.github.com>
AuthorDate: Thu Dec 24 00:47:45 2020 -0800
HDDS-4277. Support Bucket Namespace Quota Updates (#1706)
* HDDS-4277. Bucket Namespace: add namespaceQuotaUsage and update it when create and delete key in a bucket
---
.../apache/hadoop/ozone/client/OzoneBucket.java | 12 ++++++-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 2 ++
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 28 ++++++++++++++-
.../client/rpc/TestOzoneRpcClientAbstract.java | 42 ++++++++++++++++++++++
.../src/main/proto/OmClientProtocol.proto | 1 +
.../ozone/om/request/file/OMFileCreateRequest.java | 3 ++
.../ozone/om/request/key/OMKeyCreateRequest.java | 3 ++
.../ozone/om/request/key/OMKeyDeleteRequest.java | 1 +
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 19 ++++++++++
.../ozone/om/request/key/OMKeysDeleteRequest.java | 1 +
10 files changed, 110 insertions(+), 2 deletions(-)
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
index 9dcbcfc..cebe0d9 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
@@ -101,6 +101,11 @@ public class OzoneBucket extends WithMetadata {
private long usedBytes;
/**
+ * Used namespace of the bucket.
+ */
+ private long usedNamespace;
+
+ /**
* Creation time of the bucket.
*/
private Instant creationTime;
@@ -197,10 +202,11 @@ public class OzoneBucket extends WithMetadata {
Boolean versioning, long creationTime, long modificationTime,
Map<String, String> metadata, String encryptionKeyName,
String sourceVolume, String sourceBucket, long usedBytes,
- long quotaInBytes, long quotaInNamespace) {
+ long usedNamespace, long quotaInBytes, long quotaInNamespace) {
this(conf, proxy, volumeName, bucketName, storageType, versioning,
creationTime, metadata, encryptionKeyName, sourceVolume, sourceBucket);
this.usedBytes = usedBytes;
+ this.usedNamespace = usedNamespace;
this.modificationTime = Instant.ofEpochMilli(modificationTime);
this.quotaInBytes = quotaInBytes;
this.quotaInNamespace = quotaInNamespace;
@@ -509,6 +515,10 @@ public class OzoneBucket extends WithMetadata {
return usedBytes;
}
+ public long getUsedNamespace() {
+ return usedNamespace;
+ }
+
/**
* Returns Iterator to iterate over all keys in the bucket.
* The result can be restricted using key prefix, will return all
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 7472a40..28f9a01 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -628,6 +628,7 @@ public class RpcClient implements ClientProtocol {
bucketInfo.getSourceVolume(),
bucketInfo.getSourceBucket(),
bucketInfo.getUsedBytes(),
+ bucketInfo.getUsedNamespace(),
bucketInfo.getQuotaInBytes(),
bucketInfo.getQuotaInNamespace()
);
@@ -655,6 +656,7 @@ public class RpcClient implements ClientProtocol {
bucket.getSourceVolume(),
bucket.getSourceBucket(),
bucket.getUsedBytes(),
+ bucket.getUsedNamespace(),
bucket.getQuotaInBytes(),
bucket.getQuotaInNamespace()))
.collect(Collectors.toList());
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java
index 6474952..ca4bdb0 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java
@@ -81,6 +81,8 @@ public final class OmBucketInfo extends WithObjectID implements Auditable {
private long usedBytes;
+ private long usedNamespace;
+
private long quotaInBytes;
private long quotaInNamespace;
@@ -116,6 +118,7 @@ public final class OmBucketInfo extends WithObjectID implements Auditable {
String sourceVolume,
String sourceBucket,
long usedBytes,
+ long usedNamespace,
long quotaInBytes,
long quotaInNamespace) {
this.volumeName = volumeName;
@@ -132,6 +135,7 @@ public final class OmBucketInfo extends WithObjectID implements Auditable {
this.sourceVolume = sourceVolume;
this.sourceBucket = sourceBucket;
this.usedBytes = usedBytes;
+ this.usedNamespace = usedNamespace;
this.quotaInBytes = quotaInBytes;
this.quotaInNamespace = quotaInNamespace;
}
@@ -244,10 +248,18 @@ public final class OmBucketInfo extends WithObjectID implements Auditable {
return usedBytes;
}
+ public long getUsedNamespace() {
+ return usedNamespace;
+ }
+
public void incrUsedBytes(long bytes) {
this.usedBytes += bytes;
}
+ public void incrUsedNamespace(long namespaceToUse) {
+ this.usedNamespace += namespaceToUse;
+ }
+
public long getQuotaInBytes() {
return quotaInBytes;
}
@@ -292,6 +304,8 @@ public final class OmBucketInfo extends WithObjectID implements Auditable {
auditMap.put(OzoneConsts.SOURCE_BUCKET, sourceBucket);
}
auditMap.put(OzoneConsts.USED_BYTES, String.valueOf(this.usedBytes));
+ auditMap.put(OzoneConsts.USED_NAMESPACE,
+ String.valueOf(this.usedNamespace));
return auditMap;
}
@@ -329,6 +343,7 @@ public final class OmBucketInfo extends WithObjectID implements Auditable {
.setAcls(acls)
.addAllMetadata(metadata)
.setUsedBytes(usedBytes)
+ .setUsedNamespace(usedNamespace)
.setQuotaInBytes(quotaInBytes)
.setQuotaInNamespace(quotaInNamespace);
}
@@ -351,6 +366,7 @@ public final class OmBucketInfo extends WithObjectID implements Auditable {
private String sourceVolume;
private String sourceBucket;
private long usedBytes;
+ private long usedNamespace;
private long quotaInBytes;
private long quotaInNamespace;
@@ -451,6 +467,11 @@ public final class OmBucketInfo extends WithObjectID implements Auditable {
return this;
}
+ public Builder setUsedNamespace(long quotaUsage) {
+ this.usedNamespace = quotaUsage;
+ return this;
+ }
+
public Builder setQuotaInBytes(long quota) {
this.quotaInBytes = quota;
return this;
@@ -475,7 +496,7 @@ public final class OmBucketInfo extends WithObjectID implements Auditable {
return new OmBucketInfo(volumeName, bucketName, acls, isVersionEnabled,
storageType, creationTime, modificationTime, objectID, updateID,
metadata, bekInfo, sourceVolume, sourceBucket, usedBytes,
- quotaInBytes, quotaInNamespace);
+ usedNamespace, quotaInBytes, quotaInNamespace);
}
}
@@ -494,6 +515,7 @@ public final class OmBucketInfo extends WithObjectID implements Auditable {
.setObjectID(objectID)
.setUpdateID(updateID)
.setUsedBytes(usedBytes)
+ .setUsedNamespace(usedNamespace)
.addAllMetadata(KeyValueUtil.toProtobuf(metadata))
.setQuotaInBytes(quotaInBytes)
.setQuotaInNamespace(quotaInNamespace);
@@ -526,6 +548,7 @@ public final class OmBucketInfo extends WithObjectID implements Auditable {
.setUsedBytes(bucketInfo.getUsedBytes())
.setModificationTime(bucketInfo.getModificationTime())
.setQuotaInBytes(bucketInfo.getQuotaInBytes())
+ .setUsedNamespace(bucketInfo.getUsedNamespace())
.setQuotaInNamespace(bucketInfo.getQuotaInNamespace());
if (bucketInfo.hasObjectID()) {
obib.setObjectID(bucketInfo.getObjectID());
@@ -562,6 +585,7 @@ public final class OmBucketInfo extends WithObjectID implements Auditable {
", storageType='" + storageType + "'" +
", creationTime='" + creationTime + "'" +
", usedBytes='" + usedBytes + "'" +
+ ", usedNamespace='" + usedNamespace + "'" +
", quotaInBytes='" + quotaInBytes + "'" +
", quotaInNamespace='" + quotaInNamespace + '\'' +
sourceInfo +
@@ -587,6 +611,7 @@ public final class OmBucketInfo extends WithObjectID implements Auditable {
objectID == that.objectID &&
updateID == that.updateID &&
usedBytes == that.usedBytes &&
+ usedNamespace == that.usedNamespace &&
Objects.equals(sourceVolume, that.sourceVolume) &&
Objects.equals(sourceBucket, that.sourceBucket) &&
Objects.equals(metadata, that.metadata) &&
@@ -614,6 +639,7 @@ public final class OmBucketInfo extends WithObjectID implements Auditable {
", updateID=" + updateID +
", metadata=" + metadata +
", usedBytes=" + usedBytes +
+ ", usedNamespace=" + usedNamespace +
", quotaInBytes=" + quotaInBytes +
", quotaInNamespace=" + quotaInNamespace +
'}';
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
index 25d2f76..2776f59 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
@@ -935,6 +935,48 @@ public abstract class TestOzoneRpcClientAbstract {
Assert.assertEquals(0L, volume.getUsedNamespace());
}
+ @Test
+ public void testBucketUsedNamespace() throws IOException {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String key1 = UUID.randomUUID().toString();
+ String key2 = UUID.randomUUID().toString();
+ String key3 = UUID.randomUUID().toString();
+ OzoneVolume volume = null;
+ OzoneBucket bucket = null;
+
+ String value = "sample value";
+
+ store.createVolume(volumeName);
+ volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ bucket = volume.getBucket(bucketName);
+ bucket.setQuota(OzoneQuota.parseQuota(Long.MAX_VALUE + " Bytes", 2));
+
+ writeKey(bucket, key1, ONE, value, value.length());
+ Assert.assertEquals(1L,
+ store.getVolume(volumeName).getBucket(bucketName).getUsedNamespace());
+
+ writeKey(bucket, key2, ONE, value, value.length());
+ Assert.assertEquals(2L,
+ store.getVolume(volumeName).getBucket(bucketName).getUsedNamespace());
+
+ try {
+ writeKey(bucket, key3, ONE, value, value.length());
+ Assert.fail("Write key should be failed");
+ } catch (IOException ex) {
+ GenericTestUtils.assertExceptionContains("QUOTA_EXCEEDED", ex);
+ }
+
+ // Write failed, bucket usedNamespace should remain as 2
+ Assert.assertEquals(2L,
+ store.getVolume(volumeName).getBucket(bucketName).getUsedNamespace());
+
+ bucket.deleteKeys(Arrays.asList(key1, key2));
+ Assert.assertEquals(0L,
+ store.getVolume(volumeName).getBucket(bucketName).getUsedNamespace());
+ }
+
private void writeKey(OzoneBucket bucket, String keyName,
ReplicationFactor replication, String value, int valueLength)
throws IOException{
diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 3be9f77..694bc77 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -505,6 +505,7 @@ message BucketInfo {
optional uint64 usedBytes = 14;
optional int64 quotaInBytes = 15 [default = -2];
optional int64 quotaInNamespace = 16 [default = -2];
+ optional uint64 usedNamespace = 17;
}
enum StorageTypeProto {
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
index 4917595..e6f0c29 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
@@ -283,6 +283,7 @@ public class OMFileCreateRequest extends OMKeyRequest {
* ozoneManager.getScmBlockSize()
* omKeyInfo.getFactor().getNumber();
checkBucketQuotaInBytes(omBucketInfo, preAllocatedSpace);
+ checkBucketQuotaInNamespace(omBucketInfo, 1L);
// Add to cache entry can be done outside of lock for this openKey.
// Even if bucket gets deleted, when commitKey we shall identify if
@@ -298,6 +299,8 @@ public class OMFileCreateRequest extends OMKeyRequest {
trxnLogIndex);
omBucketInfo.incrUsedBytes(preAllocatedSpace);
+ // Update namespace quota
+ omBucketInfo.incrUsedNamespace(1L);
numMissingParents = missingParentInfos.size();
// Prepare response
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
index 70bf060..e78be7c 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
@@ -299,6 +299,7 @@ public class OMKeyCreateRequest extends OMKeyRequest {
* omKeyInfo.getFactor().getNumber();
// check bucket and volume quota
checkBucketQuotaInBytes(omBucketInfo, preAllocatedSpace);
+ checkBucketQuotaInNamespace(omBucketInfo, 1L);
// Add to cache entry can be done outside of lock for this openKey.
// Even if bucket gets deleted, when commitKey we shall identify if
@@ -308,6 +309,8 @@ public class OMKeyCreateRequest extends OMKeyRequest {
new CacheValue<>(Optional.of(omKeyInfo), trxnLogIndex));
omBucketInfo.incrUsedBytes(preAllocatedSpace);
+ // Update namespace quota
+ omBucketInfo.incrUsedNamespace(1L);
// Prepare response
omResponse.setCreateKeyResponse(CreateKeyResponse.newBuilder()
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
index 593dcec..0dc2b41 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
@@ -145,6 +145,7 @@ public class OMKeyDeleteRequest extends OMKeyRequest {
long quotaReleased = sumBlockLengths(omKeyInfo);
omBucketInfo.incrUsedBytes(-quotaReleased);
+ omBucketInfo.incrUsedNamespace(-1L);
// No need to add cache entries to delete table. As delete table will
// be used by DeleteKeyService only, not used for any client response
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
index fccc662..b0a737e 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
@@ -590,6 +590,25 @@ public abstract class OMKeyRequest extends OMClientRequest {
}
/**
+ * Check namespace quota.
+ */
+ protected void checkBucketQuotaInNamespace(OmBucketInfo omBucketInfo,
+ long allocatedNamespace) throws IOException {
+ if (omBucketInfo.getQuotaInNamespace() > OzoneConsts.QUOTA_RESET) {
+ long usedNamespace = omBucketInfo.getUsedNamespace();
+ long quotaInNamespace = omBucketInfo.getQuotaInNamespace();
+ long toUseNamespaceInTotal = usedNamespace + allocatedNamespace;
+ if (quotaInNamespace < toUseNamespaceInTotal) {
+ throw new OMException("The namespace quota of Bucket:"
+ + omBucketInfo.getBucketName() + " exceeded: quotaInNamespace: "
+ + quotaInNamespace + " but namespace consumed: "
+ + toUseNamespaceInTotal + ".",
+ OMException.ResultCodes.QUOTA_EXCEEDED);
+ }
+ }
+ }
+
+ /**
* Check directory exists. If exists return true, else false.
* @param volumeName
* @param bucketName
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java
index 0798e3e..c4bfa27 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java
@@ -169,6 +169,7 @@ public class OMKeysDeleteRequest extends OMKeyRequest {
quotaReleased += sumBlockLengths(omKeyInfo);
}
omBucketInfo.incrUsedBytes(-quotaReleased);
+ omBucketInfo.incrUsedNamespace(-1L * omKeyInfoList.size());
omClientResponse = new OMKeysDeleteResponse(omResponse
.setDeleteKeysResponse(DeleteKeysResponse.newBuilder()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org