You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ca...@apache.org on 2022/03/21 08:26:12 UTC
[ozone] branch master updated: HDDS-5867. Update quota when deleting open keys (#3206)
This is an automated email from the ASF dual-hosted git repository.
captainzmc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 416439d HDDS-5867. Update quota when deleting open keys (#3206)
416439d is described below
commit 416439d7a29228db2d61e98c7f634f550f3d750c
Author: Kaijie Chen <ch...@kaijie.org>
AuthorDate: Mon Mar 21 16:23:01 2022 +0800
HDDS-5867. Update quota when deleting open keys (#3206)
* Update usedBytes when deleting open key
---
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 11 +-
.../om/request/key/OMOpenKeysDeleteRequest.java | 42 +++--
.../om/response/key/OMOpenKeysDeleteResponse.java | 15 +-
.../ozone/om/request/OMRequestTestUtils.java | 16 ++
.../request/key/TestOMOpenKeysDeleteRequest.java | 174 ++++++++++-----------
.../response/key/TestOMOpenKeysDeleteResponse.java | 61 +++++++-
6 files changed, 206 insertions(+), 113 deletions(-)
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
index 723dac8..1b327d2 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
@@ -34,6 +34,7 @@ import com.google.common.base.Preconditions;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
+import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.OzoneConsts;
@@ -585,11 +586,15 @@ public abstract class OMKeyRequest extends OMClientRequest {
/**
* Return bucket info for the specified bucket.
*/
+ @Nullable
protected OmBucketInfo getBucketInfo(OMMetadataManager omMetadataManager,
String volume, String bucket) {
- return omMetadataManager.getBucketTable().getCacheValue(
- new CacheKey<>(omMetadataManager.getBucketKey(volume, bucket)))
- .getCacheValue();
+ String bucketKey = omMetadataManager.getBucketKey(volume, bucket);
+
+ CacheValue<OmBucketInfo> value = omMetadataManager.getBucketTable()
+ .getCacheValue(new CacheKey<>(bucketKey));
+
+ return value != null ? value.getCacheValue() : null;
}
/**
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMOpenKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMOpenKeysDeleteRequest.java
index 042b710..de794bd 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMOpenKeysDeleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMOpenKeysDeleteRequest.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
@@ -85,19 +86,25 @@ public class OMOpenKeysDeleteRequest extends OMKeyRequest {
OMClientResponse omClientResponse = null;
Result result = null;
Map<String, OmKeyInfo> deletedOpenKeys = new HashMap<>();
+ Map<String, OmBucketInfo> updatedBuckets = new HashMap<>();
try {
for (OpenKeyBucket openKeyBucket: submittedOpenKeyBuckets) {
// For each bucket where keys will be deleted from,
// get its bucket lock and update the cache accordingly.
- Map<String, OmKeyInfo> deleted = updateOpenKeyTableCache(ozoneManager,
- trxnLogIndex, openKeyBucket);
-
- deletedOpenKeys.putAll(deleted);
+ OmBucketInfo omBucketInfo = updateTableCachePerBucket(
+ ozoneManager, trxnLogIndex, openKeyBucket, deletedOpenKeys);
+
+ if (omBucketInfo != null) {
+ String bucketKey = ozoneManager.getMetadataManager()
+ .getBucketKey(openKeyBucket.getVolumeName(),
+ openKeyBucket.getBucketName());
+ updatedBuckets.put(bucketKey, omBucketInfo);
+ }
}
omClientResponse = new OMOpenKeysDeleteResponse(omResponse.build(),
- deletedOpenKeys, ozoneManager.isRatisEnabled());
+ deletedOpenKeys, updatedBuckets, ozoneManager.isRatisEnabled());
result = Result.SUCCESS;
} catch (IOException ex) {
@@ -137,20 +144,20 @@ public class OMOpenKeysDeleteRequest extends OMKeyRequest {
}
}
- private Map<String, OmKeyInfo> updateOpenKeyTableCache(
- OzoneManager ozoneManager, long trxnLogIndex, OpenKeyBucket keysPerBucket)
- throws IOException {
-
- Map<String, OmKeyInfo> deletedKeys = new HashMap<>();
+ private OmBucketInfo updateTableCachePerBucket(OzoneManager ozoneManager,
+ long trxnLogIndex, OpenKeyBucket keysPerBucket,
+ Map<String, OmKeyInfo> deletedOpenKeys) throws IOException {
boolean acquiredLock = false;
String volumeName = keysPerBucket.getVolumeName();
String bucketName = keysPerBucket.getBucketName();
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
+ OmBucketInfo omBucketInfo;
try {
acquiredLock = omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK,
volumeName, bucketName);
+ omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName);
for (OpenKey key: keysPerBucket.getKeysList()) {
String fullKeyName = omMetadataManager.getOpenKey(volumeName,
@@ -164,12 +171,17 @@ public class OMOpenKeysDeleteRequest extends OMKeyRequest {
if (omKeyInfo != null) {
// Set the UpdateID to current transactionLogIndex
omKeyInfo.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled());
- deletedKeys.put(fullKeyName, omKeyInfo);
+ deletedOpenKeys.put(fullKeyName, omKeyInfo);
- // Update table cache.
+ // Update openKeyTable cache.
omMetadataManager.getOpenKeyTable(getBucketLayout()).addCacheEntry(
- new CacheKey<>(fullKeyName),
- new CacheValue<>(Optional.absent(), trxnLogIndex));
+ new CacheKey<>(fullKeyName),
+ new CacheValue<>(Optional.absent(), trxnLogIndex));
+
+ // Release quota, update bucketTable cache implicitly
+ long quotaReleased = sumBlockLengths(omKeyInfo);
+ omBucketInfo.incrUsedBytes(-quotaReleased);
+ omBucketInfo.incrUsedNamespace(-1L);
ozoneManager.getMetrics().incNumOpenKeysDeleted();
LOG.debug("Open key {} deleted.", fullKeyName);
@@ -189,7 +201,7 @@ public class OMOpenKeysDeleteRequest extends OMKeyRequest {
}
}
- return deletedKeys;
+ return omBucketInfo;
}
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMOpenKeysDeleteResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMOpenKeysDeleteResponse.java
index 177548e..a3064b8 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMOpenKeysDeleteResponse.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMOpenKeysDeleteResponse.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.utils.db.BatchOperation;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
@@ -40,13 +41,17 @@ import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.OPEN_KEY_TABLE;
public class OMOpenKeysDeleteResponse extends AbstractOMKeyDeleteResponse {
private Map<String, OmKeyInfo> keysToDelete;
+ private Map<String, OmBucketInfo> bucketsToUpdate;
public OMOpenKeysDeleteResponse(
@Nonnull OzoneManagerProtocolProtos.OMResponse omResponse,
- @Nonnull Map<String, OmKeyInfo> keysToDelete, boolean isRatisEnabled) {
+ @Nonnull Map<String, OmKeyInfo> keysToDelete,
+ @Nonnull Map<String, OmBucketInfo> bucketsToUpdate,
+ boolean isRatisEnabled) {
super(omResponse, isRatisEnabled);
this.keysToDelete = keysToDelete;
+ this.bucketsToUpdate = bucketsToUpdate;
}
/**
@@ -67,9 +72,15 @@ public class OMOpenKeysDeleteResponse extends AbstractOMKeyDeleteResponse {
Table<String, OmKeyInfo> openKeyTable =
omMetadataManager.getOpenKeyTable(getBucketLayout());
- for (Map.Entry<String, OmKeyInfo> keyInfoPair: keysToDelete.entrySet()) {
+ for (Map.Entry<String, OmKeyInfo> keyInfoPair : keysToDelete.entrySet()) {
addDeletionToBatch(omMetadataManager, batchOperation, openKeyTable,
keyInfoPair.getKey(), keyInfoPair.getValue());
}
+
+ for (Map.Entry<String, OmBucketInfo> bucketInfoPair
+ : bucketsToUpdate.entrySet()) {
+ omMetadataManager.getBucketTable().putWithBatch(batchOperation,
+ bucketInfoPair.getKey(), bucketInfoPair.getValue());
+ }
}
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java
index 204b4aa..75b486e 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java
@@ -494,6 +494,22 @@ public final class OMRequestTestUtils {
new CacheValue<>(Optional.of(omBucketInfo), 1L));
}
+ /**
+ * Get bucket info from OM DB.
+ * @param volumeName
+ * @param bucketName
+ * @param omMetadataManager
+ */
+ public static OmBucketInfo getBucketFromDB(String volumeName,
+ String bucketName, OMMetadataManager omMetadataManager) {
+ String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
+
+ CacheValue<OmBucketInfo> value = omMetadataManager.getBucketTable()
+ .getCacheValue(new CacheKey<>(bucketKey));
+
+ return value != null ? value.getCacheValue() : null;
+ }
+
public static OzoneManagerProtocolProtos.OMRequest createBucketRequest(
String bucketName, String volumeName, boolean isVersionEnabled,
OzoneManagerProtocolProtos.StorageTypeProto storageTypeProto) {
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMOpenKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMOpenKeysDeleteRequest.java
index 958a6a0..86c58e6 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMOpenKeysDeleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMOpenKeysDeleteRequest.java
@@ -18,21 +18,19 @@
package org.apache.hadoop.ozone.om.request.key;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.UUID;
-import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
-import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.ozone.om.OMMetrics;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
-import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.junit.Assert;
import org.junit.Test;
-import com.google.common.base.Optional;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
@@ -77,21 +75,23 @@ public class TestOMOpenKeysDeleteRequest extends TestOMKeyRequest {
*/
@Test
public void testDeleteSubsetOfOpenKeys() throws Exception {
- final String volume1 = "volume1";
- final String volume2 = "bucket1";
- final String bucket1 = "volume2";
- final String bucket2 = "bucket2";
+ final String volume1 = UUID.randomUUID().toString();
+ final String volume2 = UUID.randomUUID().toString();
+ final String bucket1 = UUID.randomUUID().toString();
+ final String bucket2 = UUID.randomUUID().toString();
+ final long keySize = 100;
OpenKeyBucket v1b1KeysToDelete = makeOpenKeys(volume1, bucket1, 3);
OpenKeyBucket v1b1KeysToKeep = makeOpenKeys(volume1, bucket1, 3);
OpenKeyBucket v1b2KeysToDelete = makeOpenKeys(volume1, bucket2, 3);
- OpenKeyBucket v1b2KeysToKeep = makeOpenKeys(volume1, bucket2, 3);
+ OpenKeyBucket v1b2KeysToKeep = makeOpenKeys(volume1, bucket2, 2);
- OpenKeyBucket v2b2KeysToDelete = makeOpenKeys(volume2, bucket2, 3);
+ OpenKeyBucket v2b2KeysToDelete = makeOpenKeys(volume2, bucket2, 2);
OpenKeyBucket v2b2KeysToKeep = makeOpenKeys(volume2, bucket2, 3);
addToOpenKeyTableDB(
+ keySize,
v1b1KeysToKeep,
v1b2KeysToKeep,
v2b2KeysToKeep,
@@ -100,6 +100,17 @@ public class TestOMOpenKeysDeleteRequest extends TestOMKeyRequest {
v2b2KeysToDelete
);
+ final int v1b1NumKeys =
+ v1b1KeysToDelete.getKeysCount() + v1b1KeysToKeep.getKeysCount();
+ final int v1b2NumKeys =
+ v1b2KeysToDelete.getKeysCount() + v1b2KeysToKeep.getKeysCount();
+ final int v2b2NumKeys =
+ v2b2KeysToDelete.getKeysCount() + v2b2KeysToKeep.getKeysCount();
+
+ assertBucketQuota(volume1, bucket1, v1b1NumKeys, v1b1NumKeys * keySize);
+ assertBucketQuota(volume1, bucket2, v1b2NumKeys, v1b2NumKeys * keySize);
+ assertBucketQuota(volume2, bucket2, v2b2NumKeys, v2b2NumKeys * keySize);
+
deleteOpenKeysFromCache(
v1b1KeysToDelete,
v1b2KeysToDelete,
@@ -117,6 +128,14 @@ public class TestOMOpenKeysDeleteRequest extends TestOMKeyRequest {
v1b2KeysToKeep,
v2b2KeysToKeep
);
+
+ final int v1b1KeysLeft = v1b1KeysToKeep.getKeysCount();
+ final int v1b2KeysLeft = v1b2KeysToKeep.getKeysCount();
+ final int v2b2KeysLeft = v2b2KeysToKeep.getKeysCount();
+
+ assertBucketQuota(volume1, bucket1, v1b1KeysLeft, v1b1KeysLeft * keySize);
+ assertBucketQuota(volume1, bucket2, v1b2KeysLeft, v1b2KeysLeft * keySize);
+ assertBucketQuota(volume2, bucket2, v2b2KeysLeft, v2b2KeysLeft * keySize);
}
/**
@@ -126,16 +145,27 @@ public class TestOMOpenKeysDeleteRequest extends TestOMKeyRequest {
*/
@Test
public void testDeleteSameKeyNameDifferentClient() throws Exception {
+ final String volume = UUID.randomUUID().toString();
+ final String bucket = UUID.randomUUID().toString();
+ final long keySize = 100;
+
OpenKeyBucket keysToKeep =
- makeOpenKeys(volumeName, bucketName, keyName, 3);
+ makeOpenKeys(volume, bucket, 3, true);
OpenKeyBucket keysToDelete =
- makeOpenKeys(volumeName, bucketName, keyName, 3);
+ makeOpenKeys(volume, bucket, 3, true);
+
+ addToOpenKeyTableDB(keySize, keysToKeep, keysToDelete);
+
+ final int numKeys = keysToDelete.getKeysCount() + keysToKeep.getKeysCount();
+ assertBucketQuota(volume, bucket, numKeys, numKeys * keySize);
- addToOpenKeyTableDB(keysToKeep, keysToDelete);
deleteOpenKeysFromCache(keysToDelete);
assertNotInOpenKeyTable(keysToDelete);
assertInOpenKeyTable(keysToKeep);
+
+ final int keysLeft = keysToKeep.getKeysCount();
+ assertBucketQuota(volume, bucket, keysLeft, keysLeft * keySize);
}
/**
@@ -157,9 +187,9 @@ public class TestOMOpenKeysDeleteRequest extends TestOMKeyRequest {
Assert.assertEquals(metrics.getNumOpenKeysDeleted(), 0);
OpenKeyBucket existentKeys =
- makeOpenKeys(volumeName, bucketName, keyName, numExistentKeys);
+ makeOpenKeys(volumeName, bucketName, numExistentKeys, true);
OpenKeyBucket nonExistentKeys =
- makeOpenKeys(volumeName, bucketName, keyName, numNonExistentKeys);
+ makeOpenKeys(volumeName, bucketName, numNonExistentKeys, true);
addToOpenKeyTableDB(existentKeys);
deleteOpenKeysFromCache(existentKeys, nonExistentKeys);
@@ -221,20 +251,24 @@ public class TestOMOpenKeysDeleteRequest extends TestOMKeyRequest {
for (OpenKeyBucket openKeyBucket: openKeys) {
String volume = openKeyBucket.getVolumeName();
String bucket = openKeyBucket.getBucketName();
+ OmBucketInfo omBucketInfo =
+ OMRequestTestUtils.getBucketFromDB(volume, bucket, omMetadataManager);
+ if (omBucketInfo == null) {
+ OMRequestTestUtils.addBucketToDB(volume, bucket, omMetadataManager);
+ omBucketInfo = OMRequestTestUtils
+ .getBucketFromDB(volume, bucket, omMetadataManager);
+ }
for (OpenKey openKey: openKeyBucket.getKeysList()) {
+ omBucketInfo.incrUsedNamespace(1);
+ OmKeyInfo keyInfo = OMRequestTestUtils.createOmKeyInfo(volume, bucket,
+ openKey.getName(), replicationType, replicationFactor);
if (keySize > 0) {
- OmKeyInfo keyInfo = OMRequestTestUtils.createOmKeyInfo(volume, bucket,
- openKey.getName(), replicationType, replicationFactor);
- OMRequestTestUtils.addKeyLocationInfo(keyInfo, 0, keySize);
-
- OMRequestTestUtils.addKeyToTable(true, false,
- keyInfo, openKey.getClientID(), 0L, omMetadataManager);
- } else {
- OMRequestTestUtils.addKeyToTable(true,
- volume, bucket, openKey.getName(), openKey.getClientID(),
- replicationType, replicationFactor, omMetadataManager);
+ OMRequestTestUtils.addKeyLocationInfo(keyInfo, 0, keySize);
+ omBucketInfo.incrUsedBytes(keySize);
}
+ OMRequestTestUtils.addKeyToTable(true, false,
+ keyInfo, openKey.getClientID(), 0L, omMetadataManager);
}
}
@@ -254,56 +288,39 @@ public class TestOMOpenKeysDeleteRequest extends TestOMKeyRequest {
* @return A list of new open keys with size {@code numKeys}.
*/
private OpenKeyBucket makeOpenKeys(String volume, String bucket,
- int numKeys) {
-
- OpenKeyBucket.Builder keysPerBucketBuilder =
- OpenKeyBucket.newBuilder()
- .setVolumeName(volume)
- .setBucketName(bucket);
-
- for (int i = 0; i < numKeys; i++) {
- String keyName = UUID.randomUUID().toString();
- long clientID = random.nextLong();
-
- OpenKey openKey = OpenKey.newBuilder()
- .setName(keyName)
- .setClientID(clientID)
- .build();
- keysPerBucketBuilder.addKeys(openKey);
- }
-
- return keysPerBucketBuilder.build();
+ int numKeys) throws IOException {
+ return makeOpenKeys(volume, bucket, numKeys, false);
}
/**
- * Constructs a list of {@link OpenKey} objects of size {@code numKeys}.
- * The keys created will all have the same volume, bucket, and
- * key names, but randomized client IDs. These keys are not added to the
+ * Constructs a list of {@link OpenKeyBucket} objects of size {@code numKeys}.
+ * The keys created will all have the same volume and bucket, but
+ * randomized key names and client IDs. These keys are not added to the
* open key table.
*
* @param volume The volume all open keys created will have.
* @param bucket The bucket all open keys created will have.
- * @param key The key name all open keys created will have.
- * @param numKeys The number of keys with randomized key names and client
- * IDs to create.
+ * @param numKeys The number of keys with randomized client IDs to create.
+ * @param fixedKeyName If set, get key name from the {@code keyName} field,
+ * otherwise, generate random key name.
* @return A list of new open keys with size {@code numKeys}.
*/
private OpenKeyBucket makeOpenKeys(String volume, String bucket,
- String key, int numKeys) {
+ int numKeys, boolean fixedKeyName) throws IOException {
OpenKeyBucket.Builder keysPerBucketBuilder =
OpenKeyBucket.newBuilder()
- .setVolumeName(volume)
- .setBucketName(bucket);
+ .setVolumeName(volume)
+ .setBucketName(bucket);
- for (int i = 0; i < numKeys; i++) {
- long clientID = random.nextLong();
+ OpenKey.Builder openKeyBuilder = OpenKey.newBuilder().setName(keyName);
- OpenKey openKey = OpenKey.newBuilder()
- .setName(key)
- .setClientID(clientID)
- .build();
- keysPerBucketBuilder.addKeys(openKey);
+ for (int i = 0; i < numKeys; i++) {
+ openKeyBuilder.setClientID(random.nextLong());
+ if (!fixedKeyName) {
+ openKeyBuilder.setName(UUID.randomUUID().toString());
+ }
+ keysPerBucketBuilder.addKeys(openKeyBuilder.build());
}
return keysPerBucketBuilder.build();
@@ -327,6 +344,15 @@ public class TestOMOpenKeysDeleteRequest extends TestOMKeyRequest {
}
}
+ private void assertBucketQuota(String volumeName, String bucketName,
+ int usedNamespace, long usedBytes) {
+ OmBucketInfo omBucketInfo = OMRequestTestUtils.getBucketFromDB(
+ volumeName, bucketName, omMetadataManager);
+ Assert.assertNotNull(omBucketInfo);
+ Assert.assertEquals(usedNamespace, omBucketInfo.getUsedNamespace());
+ Assert.assertEquals(usedBytes, omBucketInfo.getUsedBytes());
+ }
+
/**
* Expands all the open keys represented by {@code openKeyBuckets} to their
* full
@@ -388,32 +414,4 @@ public class TestOMOpenKeysDeleteRequest extends TestOMKeyRequest {
.setClientId(UUID.randomUUID().toString()).build();
}
- private void addVolumeToCacheAndDB(OmVolumeArgs volumeArgs) throws Exception {
- String volumeKey = omMetadataManager.getVolumeKey(volumeArgs.getVolume());
-
- omMetadataManager.getVolumeTable().addCacheEntry(
- new CacheKey<>(volumeKey),
- new CacheValue<>(Optional.of(volumeArgs), volumeArgs.getUpdateID())
- );
-
- omMetadataManager.getVolumeTable().put(volumeKey, volumeArgs);
- }
-
- private OmVolumeArgs getVolumeFromDB(String volume) throws Exception {
- String volumeKey = omMetadataManager.getVolumeKey(volume);
- return omMetadataManager.getVolumeTable().getSkipCache(volumeKey);
- }
-
- private OmVolumeArgs getVolumeFromCache(String volume) {
- String volumeKey = omMetadataManager.getVolumeKey(volume);
- CacheValue<OmVolumeArgs> value = omMetadataManager.getVolumeTable()
- .getCacheValue(new CacheKey<>(volumeKey));
-
- OmVolumeArgs result = null;
- if (value != null) {
- result = value.getCacheValue();
- }
-
- return result;
- }
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMOpenKeysDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMOpenKeysDeleteResponse.java
index ebe2dee..e4c44fb 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMOpenKeysDeleteResponse.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMOpenKeysDeleteResponse.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.om.response.key;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
import org.junit.Assert;
import org.junit.Test;
@@ -45,7 +46,9 @@ public class TestOMOpenKeysDeleteResponse extends TestOMKeyResponse {
public void testAddToDBBatchWithEmptyBlocks() throws Exception {
Map<String, OmKeyInfo> keysToDelete = addOpenKeysToDB(volumeName, 3);
Map<String, OmKeyInfo> keysToKeep = addOpenKeysToDB(volumeName, 3);
- createAndCommitResponse(keysToDelete, Status.OK);
+ Map<String, OmBucketInfo> bucketsToUpdate = updateBuckets(keysToDelete);
+
+ createAndCommitResponse(keysToDelete, bucketsToUpdate, Status.OK);
for (String key: keysToDelete.keySet()) {
// open keys with no associated block data should have been removed
@@ -61,6 +64,14 @@ public class TestOMOpenKeysDeleteResponse extends TestOMKeyResponse {
omMetadataManager.getOpenKeyTable(getBucketLayout()).isExist(key));
Assert.assertFalse(omMetadataManager.getDeletedTable().isExist(key));
}
+
+ for (OmBucketInfo omBucketInfo : bucketsToUpdate.values()) {
+ String bucketKey = omMetadataManager.getBucketKey(
+ omBucketInfo.getVolumeName(), omBucketInfo.getBucketName());
+ OmBucketInfo dbBucketInfo =
+ omMetadataManager.getBucketTable().get(bucketKey);
+ Assert.assertEquals(omBucketInfo, dbBucketInfo);
+ }
}
/**
@@ -73,8 +84,9 @@ public class TestOMOpenKeysDeleteResponse extends TestOMKeyResponse {
KEY_LENGTH);
Map<String, OmKeyInfo> keysToKeep = addOpenKeysToDB(volumeName, 3,
KEY_LENGTH);
+ Map<String, OmBucketInfo> bucketsToUpdate = updateBuckets(keysToDelete);
- createAndCommitResponse(keysToDelete, Status.OK);
+ createAndCommitResponse(keysToDelete, bucketsToUpdate, Status.OK);
for (String key: keysToDelete.keySet()) {
// These keys should have been moved from the open key table to the
@@ -90,6 +102,14 @@ public class TestOMOpenKeysDeleteResponse extends TestOMKeyResponse {
omMetadataManager.getOpenKeyTable(getBucketLayout()).isExist(key));
Assert.assertFalse(omMetadataManager.getDeletedTable().isExist(key));
}
+
+ for (OmBucketInfo omBucketInfo : bucketsToUpdate.values()) {
+ String bucketKey = omMetadataManager.getBucketKey(
+ omBucketInfo.getVolumeName(), omBucketInfo.getBucketName());
+ OmBucketInfo dbBucketInfo =
+ omMetadataManager.getBucketTable().get(bucketKey);
+ Assert.assertEquals(omBucketInfo, dbBucketInfo);
+ }
}
/**
@@ -100,7 +120,10 @@ public class TestOMOpenKeysDeleteResponse extends TestOMKeyResponse {
@Test
public void testAddToDBBatchWithErrorResponse() throws Exception {
Map<String, OmKeyInfo> keysToDelete = addOpenKeysToDB(volumeName, 3);
- createAndCommitResponse(keysToDelete, Status.INTERNAL_ERROR);
+ Map<String, OmBucketInfo> bucketsToUpdate = new HashMap<>();
+
+ createAndCommitResponse(keysToDelete, bucketsToUpdate,
+ Status.INTERNAL_ERROR);
for (String key: keysToDelete.keySet()) {
// If an error occurs in the response, the batch operation moving keys
@@ -119,7 +142,8 @@ public class TestOMOpenKeysDeleteResponse extends TestOMKeyResponse {
* @throws Exception
*/
private void createAndCommitResponse(Map<String, OmKeyInfo> keysToDelete,
- Status status) throws Exception {
+ Map<String, OmBucketInfo> bucketsToUpdate, Status status)
+ throws Exception {
OMResponse omResponse = OMResponse.newBuilder()
.setStatus(status)
@@ -127,7 +151,7 @@ public class TestOMOpenKeysDeleteResponse extends TestOMKeyResponse {
.build();
OMOpenKeysDeleteResponse response = new OMOpenKeysDeleteResponse(omResponse,
- keysToDelete, true);
+ keysToDelete, bucketsToUpdate, true);
// Operations are only added to the batch by this method when status is OK.
response.checkAndUpdateDB(omMetadataManager, batchOperation);
@@ -164,11 +188,21 @@ public class TestOMOpenKeysDeleteResponse extends TestOMKeyResponse {
String key = UUID.randomUUID().toString();
long clientID = random.nextLong();
+ OmBucketInfo omBucketInfo = OMRequestTestUtils.getBucketFromDB(volume,
+ bucket, omMetadataManager);
+ if (omBucketInfo == null) {
+ OMRequestTestUtils.addBucketToDB(volume, bucket, omMetadataManager);
+ omBucketInfo = OMRequestTestUtils.getBucketFromDB(volume,
+ bucket, omMetadataManager);
+ }
+
OmKeyInfo omKeyInfo = OMRequestTestUtils.createOmKeyInfo(volume,
bucket, key, replicationType, replicationFactor);
+ omBucketInfo.incrUsedNamespace(1);
if (keyLength > 0) {
OMRequestTestUtils.addKeyLocationInfo(omKeyInfo, 0, keyLength);
+ omBucketInfo.incrUsedBytes(keyLength);
}
String openKey = omMetadataManager.getOpenKey(volume, bucket,
@@ -187,4 +221,21 @@ public class TestOMOpenKeysDeleteResponse extends TestOMKeyResponse {
return newOpenKeys;
}
+
+ private Map<String, OmBucketInfo> updateBuckets(
+ Map<String, OmKeyInfo> keysToDelete) {
+ Map<String, OmBucketInfo> bucketsToUpdate = new HashMap<>();
+
+ for (OmKeyInfo omKeyInfo : keysToDelete.values()) {
+ OmBucketInfo omBucketInfo = OMRequestTestUtils.getBucketFromDB(
+ omKeyInfo.getVolumeName(), omKeyInfo.getBucketName(),
+ omMetadataManager);
+ String bucketKey = omMetadataManager.getBucketKey(
+ omKeyInfo.getVolumeName(), omKeyInfo.getBucketName());
+ omBucketInfo.incrUsedNamespace(-1);
+ omBucketInfo.incrUsedBytes(-omKeyInfo.getDataSize());
+ bucketsToUpdate.put(bucketKey, omBucketInfo);
+ }
+ return bucketsToUpdate;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org