You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ca...@apache.org on 2022/07/14 11:09:11 UTC
[ozone] branch master updated: HDDS-6998. Ignore keys with higher updateID in OpenKeyDeleteRequest (#3596)
This is an automated email from the ASF dual-hosted git repository.
captainzmc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new e1f39bd506 HDDS-6998. Ignore keys with higher updateID in OpenKeyDeleteRequest (#3596)
e1f39bd506 is described below
commit e1f39bd50684f29f6055f569d12eb317fde642b2
Author: Kaijie Chen <ch...@kaijie.org>
AuthorDate: Thu Jul 14 19:09:06 2022 +0800
HDDS-6998. Ignore keys with higher updateID in OpenKeyDeleteRequest (#3596)
* HDDS-6998. Ignore keys with higher updateID in OpenKeyDeleteRequest
---
.../om/request/key/OMOpenKeysDeleteRequest.java | 8 +++
.../request/key/TestOMOpenKeysDeleteRequest.java | 71 +++++++++++++++++++++-
2 files changed, 77 insertions(+), 2 deletions(-)
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMOpenKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMOpenKeysDeleteRequest.java
index 197da11041..d2e2545341 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMOpenKeysDeleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMOpenKeysDeleteRequest.java
@@ -159,6 +159,14 @@ public class OMOpenKeysDeleteRequest extends OMKeyRequest {
omMetadataManager.getOpenKeyTable(getBucketLayout())
.get(fullKeyName);
if (omKeyInfo != null) {
+ if (ozoneManager.isRatisEnabled() &&
+ trxnLogIndex < omKeyInfo.getUpdateID()) {
+ LOG.warn("Transaction log index {} is smaller than " +
+ "the current updateID {} of key {}, skipping deletion.",
+ trxnLogIndex, omKeyInfo.getUpdateID(), fullKeyName);
+ continue;
+ }
+
// Set the UpdateID to current transactionLogIndex
omKeyInfo.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled());
deletedOpenKeys.put(fullKeyName, omKeyInfo);
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMOpenKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMOpenKeysDeleteRequest.java
index 49daf51ec3..082b06b329 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMOpenKeysDeleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMOpenKeysDeleteRequest.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
+import org.apache.hadoop.ozone.om.helpers.WithObjectID;
import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.junit.Assert;
@@ -190,6 +191,71 @@ public class TestOMOpenKeysDeleteRequest extends TestOMKeyRequest {
assertInOpenKeyTable(keysToKeep);
}
+ /**
+ * Tests removing keys from the open key table cache that have higher
+ * updateID than the transactionID. Those keys should be ignored.
+ * It is OK if updateID equals to or less than transactionID.
+ * See {@link WithObjectID#setUpdateID(long, boolean)}.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testDeleteKeyWithHigherUpdateID() throws Exception {
+ final String volume = UUID.randomUUID().toString();
+ final String bucket = UUID.randomUUID().toString();
+
+ OMRequestTestUtils.addVolumeAndBucketToDB(volume, bucket,
+ omMetadataManager, getBucketLayout());
+
+ final long updateId = 200L;
+ final long transactionId = 100L;
+
+ OmKeyInfo.Builder builder = new OmKeyInfo.Builder()
+ .setVolumeName(volume)
+ .setBucketName(bucket)
+ .setUpdateID(updateId)
+ .setReplicationConfig(ReplicationConfig.fromTypeAndFactor(
+ ReplicationType.RATIS, ReplicationFactor.THREE));
+
+ if (getBucketLayout().isFileSystemOptimized()) {
+ builder.setParentObjectID(random.nextLong());
+ }
+
+ List<Pair<Long, OmKeyInfo>> keysWithHigherUpdateID = new ArrayList<>(1);
+ keysWithHigherUpdateID.add(Pair.of(clientID,
+ builder.setKeyName("key")
+ .setFileName("key")
+ .setUpdateID(updateId)
+ .build()));
+
+ List<Pair<Long, OmKeyInfo>> keysWithSameUpdateID = new ArrayList<>(1);
+ keysWithSameUpdateID.add(Pair.of(clientID,
+ builder.setKeyName("key2")
+ .setFileName("key2")
+ .setUpdateID(transactionId)
+ .build()));
+
+ List<Pair<Long, OmKeyInfo>> allKeys = new ArrayList<>(2);
+ allKeys.addAll(keysWithHigherUpdateID);
+ allKeys.addAll(keysWithSameUpdateID);
+
+ addToOpenKeyTableDB(allKeys);
+
+ OMRequest omRequest = doPreExecute(createDeleteOpenKeyRequest(allKeys));
+ OMOpenKeysDeleteRequest openKeyDeleteRequest =
+ new OMOpenKeysDeleteRequest(omRequest, getBucketLayout());
+
+ OMClientResponse omClientResponse =
+ openKeyDeleteRequest.validateAndUpdateCache(ozoneManager,
+ transactionId, ozoneManagerDoubleBufferHelper);
+
+ Assert.assertEquals(Status.OK,
+ omClientResponse.getOMResponse().getStatus());
+
+ assertInOpenKeyTable(keysWithHigherUpdateID);
+ assertNotInOpenKeyTable(keysWithSameUpdateID);
+ }
+
/**
* Tests metrics set by {@link OMOpenKeysDeleteRequest}.
* Submits a set of keys for deletion where only some of the keys actually
@@ -289,10 +355,11 @@ public class TestOMOpenKeysDeleteRequest extends TestOMKeyRequest {
if (getBucketLayout().isFileSystemOptimized()) {
OMRequestTestUtils.addFileToKeyTable(
true, false, omKeyInfo.getFileName(),
- omKeyInfo, clientID, 0L, omMetadataManager);
+ omKeyInfo, clientID, omKeyInfo.getUpdateID(), omMetadataManager);
} else {
OMRequestTestUtils.addKeyToTable(
- true, false, omKeyInfo, clientID, 0L, omMetadataManager);
+ true, false,
+ omKeyInfo, clientID, omKeyInfo.getUpdateID(), omMetadataManager);
}
}
assertInOpenKeyTable(openKeys);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org