You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@ozone.apache.org by "hemantk-12 (via GitHub)" <gi...@apache.org> on 2023/06/29 00:07:19 UTC
[GitHub] [ozone] hemantk-12 commented on a diff in pull request #4187: HDDS-7798. [Snapshot] SnapDiff : Edge case handling Unit Tests
hemantk-12 commented on code in PR #4187:
URL: https://github.com/apache/ozone/pull/4187#discussion_r1245916565
##########
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java:
##########
@@ -837,4 +837,25 @@ public String getPath() {
}
return getParentObjectID() + OzoneConsts.OM_KEY_PREFIX + getFileName();
}
+
+ @Override
+ public String toString() {
+ return "OmKeyInfo{" +
+ "volumeName='" + volumeName + '\'' +
+ ", bucketName='" + bucketName + '\'' +
+ ", keyName='" + keyName + '\'' +
Review Comment:
Curious why volumeName, bucketName and keyName are enclosed with `'` but not other parameters.
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java:
##########
@@ -547,6 +572,588 @@ public void testCreateSnapshotMissingMandatoryParams() throws Exception {
() -> createSnapshot(nullstr, bucket));
}
+ private Set<OmKeyInfo> getDeletedKeysFromRocksDb(
+ OMMetadataManager metadataManager) throws IOException {
+ Set<OmKeyInfo> deletedKeys = Sets.newHashSet();
+ try (TableIterator<String,
+ ? extends Table.KeyValue<String, RepeatedOmKeyInfo>>
+ deletedTableIterator = metadataManager.getDeletedTable()
+ .iterator()) {
+ while (deletedTableIterator.hasNext()) {
+ Table.KeyValue<String, RepeatedOmKeyInfo> val =
+ deletedTableIterator.next();
+ deletedKeys.addAll(val.getValue().getOmKeyInfoList());
+ }
+ }
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
+ deletedDirTableIterator = metadataManager.getDeletedDirTable()
+ .iterator()) {
+ while (deletedDirTableIterator.hasNext()) {
+ deletedKeys.add(deletedDirTableIterator.next().getValue());
+ }
+ }
+ return deletedKeys;
+ }
+
+ private OmKeyInfo getOmKeyInfo(String volume, String bucket,
+ String key) throws IOException {
+ return cluster.getOzoneManager().getKeyManager()
+ .getKeyInfo(new OmKeyArgs.Builder().setVolumeName(volume)
+ .setBucketName(bucket).setKeyName(key).build(), null);
+ }
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Snapshot snap2 is created
+ * 4) Key k1 is deleted.
+ * 5) Snapdiff b/w snap3 & snap2 taken to assert difference of 1 key
Review Comment:
Please add when snap3 was taken.
##########
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java:
##########
@@ -987,14 +987,16 @@ public SnapshotDiffResponse snapshotDiff(String volumeName,
String toSnapshot,
String token,
int pageSize,
- boolean forceFullDiff)
+ boolean forceFullDiff,
+ boolean forceNonNativeDiff)
Review Comment:
May be `skipNativeDiff` or something simpler instead of double negation.
##########
hadoop-ozone/s3gateway/pom.xml:
##########
@@ -177,6 +177,10 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
Review Comment:
Why do you needed this? Is it just to have the ability to write tests in Junit-5?
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java:
##########
@@ -547,6 +572,588 @@ public void testCreateSnapshotMissingMandatoryParams() throws Exception {
() -> createSnapshot(nullstr, bucket));
}
+ private Set<OmKeyInfo> getDeletedKeysFromRocksDb(
+ OMMetadataManager metadataManager) throws IOException {
+ Set<OmKeyInfo> deletedKeys = Sets.newHashSet();
+ try (TableIterator<String,
+ ? extends Table.KeyValue<String, RepeatedOmKeyInfo>>
+ deletedTableIterator = metadataManager.getDeletedTable()
+ .iterator()) {
+ while (deletedTableIterator.hasNext()) {
+ Table.KeyValue<String, RepeatedOmKeyInfo> val =
+ deletedTableIterator.next();
+ deletedKeys.addAll(val.getValue().getOmKeyInfoList());
+ }
+ }
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
+ deletedDirTableIterator = metadataManager.getDeletedDirTable()
+ .iterator()) {
+ while (deletedDirTableIterator.hasNext()) {
+ deletedKeys.add(deletedDirTableIterator.next().getValue());
+ }
+ }
+ return deletedKeys;
+ }
+
+ private OmKeyInfo getOmKeyInfo(String volume, String bucket,
+ String key) throws IOException {
+ return cluster.getOzoneManager().getKeyManager()
+ .getKeyInfo(new OmKeyArgs.Builder().setVolumeName(volume)
+ .setBucketName(bucket).setKeyName(key).build(), null);
+ }
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Snapshot snap2 is created
+ * 4) Key k1 is deleted.
+ * 5) Snapdiff b/w snap3 & snap2 taken to assert difference of 1 key
+ */
+ @Test
+ public void testSnapDiffHandlingReclaimWithLatestUse() throws Exception {
+ String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5);
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ volume.createBucket(testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+ OmKeyInfo keyInfo = getOmKeyInfo(testVolumeName, testBucketName, key1);
+ bucket.deleteKey(key1);
+ String snap3 = "snap3";
+ String activeSnapshotPrefix =
+ createSnapshot(testVolumeName, testBucketName, snap3);
+ SnapshotDiffReportOzone diff =
+ getSnapDiffReport(testVolumeName, testBucketName, snap1, snap2);
+ Assert.assertEquals(diff.getDiffList().size(), 0);
+ diff = getSnapDiffReport(testVolumeName, testBucketName, snap2, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 1);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1)));
+ }
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Key k1 is deleted.
+ * 4) Snapshot snap2 is created.
+ * 5) Snapshot snap3 is created.
+ * 6) Snapdiff b/w snap3 & snap1 taken to assert difference of 1 key.
+ * 7) Snapdiff b/w snap3 & snap2 taken to assert difference of 0 key.
+ */
+ @Test
+ public void testSnapDiffHandlingReclaimWithPreviousUse() throws Exception {
+ String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5);
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ volume.createBucket(testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ OmKeyInfo keyInfo = getOmKeyInfo(testVolumeName, testBucketName, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ bucket.deleteKey(key1);
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+ String snap3 = "snap3";
+ String activeSnapshotPrefix = createSnapshot(testVolumeName, testBucketName,
+ snap3);
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
+ testBucketName, snap1, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 1);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1)));
+ diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap2, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 0);
+ }
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Key k1 is deleted.
+ * 4) Key k1 is recreated.
+ * 5) Snapshot snap2 is created.
+ * 6) Snapdiff b/w Active FS & snap1 taken to assert difference of 2 keys.
+ * 7) Snapdiff b/w Active FS & snap2 taken to assert difference of 0 key.
+ * 8) Checking rocks db to ensure the object created shouldn't be reclaimed
+ * as it is used by snapshot.
+ * 9) Key k1 is deleted.
+ * 10) Snapdiff b/w Active FS & snap1 taken to assert difference of 1 key.
+ * 11) Snapdiff b/w Active FS & snap2 taken to assert difference of 1 key.
+ */
+ @Test
+ public void testSnapDiffReclaimWithKeyRecreation() throws Exception {
+ String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5);
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ volume.createBucket(testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ OmKeyInfo keyInfo1 = getOmKeyInfo(testVolumeName, testBucketName, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ bucket.deleteKey(key1);
+ key1 = createFileKey(bucket, key1);
+ OmKeyInfo keyInfo2 = getOmKeyInfo(testVolumeName, testBucketName, key1);
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+ String snap3 = "snap3";
+ String activeSnapshotPrefix = createSnapshot(testVolumeName, testBucketName,
+ snap3);
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
+ testBucketName, snap1, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 2);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1),
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.CREATE, OZONE_URI_DELIMITER + key1)));
+ diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap2, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 0);
+ bucket.deleteKey(key1);
+ String snap4 = "snap4";
+ activeSnapshotPrefix = createSnapshot(testVolumeName, testBucketName,
+ snap4);
+ diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap1, snap4);
+ Assert.assertEquals(diff.getDiffList().size(), 1);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1)));
+ diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap2, snap4);
+ Assert.assertEquals(diff.getDiffList().size(), 1);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1)));
+ }
+
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Key k1 is renamed to renamed-k1.
+ * 4) Key renamed-k1 is deleted.
+ * 5) Snapshot snap2 created.
+ * 4) Snapdiff b/w snap2 & snap1 taken to assert difference of 1 key.
+ */
+ @Test
+ public void testSnapDiffReclaimWithKeyRename() throws Exception {
+ String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5);
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ volume.createBucket(testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ String renamedKey = "renamed-" + key1;
+ bucket.renameKey(key1, renamedKey);
+ GenericTestUtils.waitFor(
+ () -> {
+ try {
+ getOmKeyInfo(testVolumeName, testBucketName, renamedKey);
+ } catch (IOException e) {
+ return false;
+ }
+ return true;
+ }, 1000, 10000);
Review Comment:
```suggestion
GenericTestUtils.waitFor(
() -> {
try {
getOmKeyInfo(testVolumeName, testBucketName, renamedKey);
} catch (IOException e) {
return false;
}
return true;
}, 1000, 10000);
```
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java:
##########
@@ -547,6 +572,588 @@ public void testCreateSnapshotMissingMandatoryParams() throws Exception {
() -> createSnapshot(nullstr, bucket));
}
+ private Set<OmKeyInfo> getDeletedKeysFromRocksDb(
+ OMMetadataManager metadataManager) throws IOException {
+ Set<OmKeyInfo> deletedKeys = Sets.newHashSet();
+ try (TableIterator<String,
+ ? extends Table.KeyValue<String, RepeatedOmKeyInfo>>
+ deletedTableIterator = metadataManager.getDeletedTable()
+ .iterator()) {
+ while (deletedTableIterator.hasNext()) {
+ Table.KeyValue<String, RepeatedOmKeyInfo> val =
+ deletedTableIterator.next();
+ deletedKeys.addAll(val.getValue().getOmKeyInfoList());
+ }
+ }
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
+ deletedDirTableIterator = metadataManager.getDeletedDirTable()
+ .iterator()) {
+ while (deletedDirTableIterator.hasNext()) {
+ deletedKeys.add(deletedDirTableIterator.next().getValue());
+ }
+ }
+ return deletedKeys;
+ }
+
+ private OmKeyInfo getOmKeyInfo(String volume, String bucket,
+ String key) throws IOException {
+ return cluster.getOzoneManager().getKeyManager()
+ .getKeyInfo(new OmKeyArgs.Builder().setVolumeName(volume)
+ .setBucketName(bucket).setKeyName(key).build(), null);
+ }
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Snapshot snap2 is created
+ * 4) Key k1 is deleted.
+ * 5) Snapdiff b/w snap3 & snap2 taken to assert difference of 1 key
+ */
+ @Test
+ public void testSnapDiffHandlingReclaimWithLatestUse() throws Exception {
+ String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5);
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ volume.createBucket(testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+ OmKeyInfo keyInfo = getOmKeyInfo(testVolumeName, testBucketName, key1);
+ bucket.deleteKey(key1);
+ String snap3 = "snap3";
+ String activeSnapshotPrefix =
+ createSnapshot(testVolumeName, testBucketName, snap3);
+ SnapshotDiffReportOzone diff =
+ getSnapDiffReport(testVolumeName, testBucketName, snap1, snap2);
+ Assert.assertEquals(diff.getDiffList().size(), 0);
+ diff = getSnapDiffReport(testVolumeName, testBucketName, snap2, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 1);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1)));
+ }
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Key k1 is deleted.
+ * 4) Snapshot snap2 is created.
+ * 5) Snapshot snap3 is created.
+ * 6) Snapdiff b/w snap3 & snap1 taken to assert difference of 1 key.
+ * 7) Snapdiff b/w snap3 & snap2 taken to assert difference of 0 key.
+ */
+ @Test
+ public void testSnapDiffHandlingReclaimWithPreviousUse() throws Exception {
+ String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5);
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ volume.createBucket(testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ OmKeyInfo keyInfo = getOmKeyInfo(testVolumeName, testBucketName, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ bucket.deleteKey(key1);
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+ String snap3 = "snap3";
+ String activeSnapshotPrefix = createSnapshot(testVolumeName, testBucketName,
+ snap3);
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
+ testBucketName, snap1, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 1);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1)));
+ diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap2, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 0);
+ }
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Key k1 is deleted.
+ * 4) Key k1 is recreated.
+ * 5) Snapshot snap2 is created.
+ * 6) Snapdiff b/w Active FS & snap1 taken to assert difference of 2 keys.
+ * 7) Snapdiff b/w Active FS & snap2 taken to assert difference of 0 key.
+ * 8) Checking rocks db to ensure the object created shouldn't be reclaimed
+ * as it is used by snapshot.
+ * 9) Key k1 is deleted.
+ * 10) Snapdiff b/w Active FS & snap1 taken to assert difference of 1 key.
+ * 11) Snapdiff b/w Active FS & snap2 taken to assert difference of 1 key.
+ */
+ @Test
+ public void testSnapDiffReclaimWithKeyRecreation() throws Exception {
+ String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5);
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ volume.createBucket(testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ OmKeyInfo keyInfo1 = getOmKeyInfo(testVolumeName, testBucketName, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ bucket.deleteKey(key1);
+ key1 = createFileKey(bucket, key1);
+ OmKeyInfo keyInfo2 = getOmKeyInfo(testVolumeName, testBucketName, key1);
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+ String snap3 = "snap3";
+ String activeSnapshotPrefix = createSnapshot(testVolumeName, testBucketName,
+ snap3);
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
+ testBucketName, snap1, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 2);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1),
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.CREATE, OZONE_URI_DELIMITER + key1)));
+ diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap2, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 0);
+ bucket.deleteKey(key1);
+ String snap4 = "snap4";
+ activeSnapshotPrefix = createSnapshot(testVolumeName, testBucketName,
+ snap4);
+ diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap1, snap4);
+ Assert.assertEquals(diff.getDiffList().size(), 1);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1)));
+ diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap2, snap4);
+ Assert.assertEquals(diff.getDiffList().size(), 1);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1)));
+ }
+
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Key k1 is renamed to renamed-k1.
+ * 4) Key renamed-k1 is deleted.
+ * 5) Snapshot snap2 created.
+ * 4) Snapdiff b/w snap2 & snap1 taken to assert difference of 1 key.
+ */
+ @Test
+ public void testSnapDiffReclaimWithKeyRename() throws Exception {
+ String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5);
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ volume.createBucket(testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ String renamedKey = "renamed-" + key1;
+ bucket.renameKey(key1, renamedKey);
+ GenericTestUtils.waitFor(
+ () -> {
+ try {
+ getOmKeyInfo(testVolumeName, testBucketName, renamedKey);
+ } catch (IOException e) {
+ return false;
+ }
+ return true;
+ }, 1000, 10000);
+ OmKeyInfo renamedKeyInfo = getOmKeyInfo(testVolumeName, testBucketName,
+ renamedKey);
+ bucket.deleteKey(renamedKey);
+ String snap2 = "snap2";
+ String activeSnapshotPrefix = createSnapshot(testVolumeName, testBucketName,
+ snap2);
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
+ testBucketName, snap1, snap2);
+ Assert.assertEquals(diff.getDiffList().size(), 1);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1)
+ ));
+ }
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Key k1 is renamed to renamed-k1.
+ * 4) Key renamed-k1 is renamed to renamed-renamed-k1.
+ * 5) Key renamed-renamed-k1 is deleted.
+ * 6) Snapshot snap2 is created.
+ * 7) Snapdiff b/w Active FS & snap1 taken to assert difference of 1 key.
+ */
+ @Test
+ public void testSnapDiffWith2RenamesAndDelete() throws Exception {
+ String testVolumeName = "vol" + counter.incrementAndGet();
+ String testBucketName = "bucket" + counter.incrementAndGet();
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ volume.createBucket(testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ String renamedKey = "renamed-" + key1;
+ bucket.renameKey(key1, renamedKey);
+ GenericTestUtils.waitFor(
Review Comment:
Same as above.
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java:
##########
@@ -547,6 +572,588 @@ public void testCreateSnapshotMissingMandatoryParams() throws Exception {
() -> createSnapshot(nullstr, bucket));
}
+ private Set<OmKeyInfo> getDeletedKeysFromRocksDb(
+ OMMetadataManager metadataManager) throws IOException {
+ Set<OmKeyInfo> deletedKeys = Sets.newHashSet();
+ try (TableIterator<String,
+ ? extends Table.KeyValue<String, RepeatedOmKeyInfo>>
+ deletedTableIterator = metadataManager.getDeletedTable()
+ .iterator()) {
+ while (deletedTableIterator.hasNext()) {
+ Table.KeyValue<String, RepeatedOmKeyInfo> val =
+ deletedTableIterator.next();
+ deletedKeys.addAll(val.getValue().getOmKeyInfoList());
+ }
+ }
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
+ deletedDirTableIterator = metadataManager.getDeletedDirTable()
+ .iterator()) {
+ while (deletedDirTableIterator.hasNext()) {
+ deletedKeys.add(deletedDirTableIterator.next().getValue());
+ }
+ }
+ return deletedKeys;
+ }
+
+ private OmKeyInfo getOmKeyInfo(String volume, String bucket,
+ String key) throws IOException {
+ return cluster.getOzoneManager().getKeyManager()
+ .getKeyInfo(new OmKeyArgs.Builder().setVolumeName(volume)
+ .setBucketName(bucket).setKeyName(key).build(), null);
+ }
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Snapshot snap2 is created
+ * 4) Key k1 is deleted.
+ * 5) Snapdiff b/w snap3 & snap2 taken to assert difference of 1 key
+ */
+ @Test
+ public void testSnapDiffHandlingReclaimWithLatestUse() throws Exception {
Review Comment:
General comment about tests, I feel there is lots of repetitive code which can be avoid using helper functions or reusing setup.
The other problem is the readability. Even tho you did a good job in adding tests for all the edge cases, one can't be certain what all is covered. It would be great if you describe it in some tableur format. Doesn't have to be part of this PR.
##########
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java:
##########
@@ -668,19 +668,22 @@ private List<OzoneSnapshot> getNextListOfSnapshots(String prevSnapshot)
* @param token to get the index to return diff report from.
* @param pageSize maximum entries returned to the report.
* @param forceFullDiff request to force full diff, skipping DAG optimization
+ * @param forceNonNativeDiff request to force diff to not
Review Comment:
`request to force diff to not ...` ? not what?
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java:
##########
@@ -1060,7 +1079,7 @@ private void getDeltaFilesAndDiffKeysToObjectIdToKeyMap(
addToObjectIdMap(fsTable,
tsTable,
deltaFiles,
- sstDumpTool.isPresent(),
+ forceNonNativeDiff || !sstDumpTool.isPresent(),
Review Comment:
1. Did you mean `forceNonNativeDiff || sstDumpTool.isPresent()`? If not, I don't think it would even run when you pass `forceNonNativeDiff=true` and sstDumpTool is not present. Same for when sstDumpTool is not preset and forceNonNativeDiff=false, result will be true but it will still load native lib because of this check: https://github.com/apache/ozone/blob/0862a76ef178f427a272c335374760c87c885119/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java#L1095
##########
hadoop-ozone/ozone-manager/pom.xml:
##########
@@ -262,6 +262,14 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<groupId>org.mockito</groupId>
<artifactId>mockito-junit-jupiter</artifactId>
</dependency>
+ <dependency>
Review Comment:
I don't see any use of these. If I'm not wrong, please remove these.
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java:
##########
@@ -547,6 +572,588 @@ public void testCreateSnapshotMissingMandatoryParams() throws Exception {
() -> createSnapshot(nullstr, bucket));
}
+ private Set<OmKeyInfo> getDeletedKeysFromRocksDb(
+ OMMetadataManager metadataManager) throws IOException {
+ Set<OmKeyInfo> deletedKeys = Sets.newHashSet();
+ try (TableIterator<String,
+ ? extends Table.KeyValue<String, RepeatedOmKeyInfo>>
+ deletedTableIterator = metadataManager.getDeletedTable()
+ .iterator()) {
+ while (deletedTableIterator.hasNext()) {
+ Table.KeyValue<String, RepeatedOmKeyInfo> val =
+ deletedTableIterator.next();
+ deletedKeys.addAll(val.getValue().getOmKeyInfoList());
+ }
+ }
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
+ deletedDirTableIterator = metadataManager.getDeletedDirTable()
+ .iterator()) {
+ while (deletedDirTableIterator.hasNext()) {
+ deletedKeys.add(deletedDirTableIterator.next().getValue());
+ }
+ }
+ return deletedKeys;
+ }
+
+ private OmKeyInfo getOmKeyInfo(String volume, String bucket,
+ String key) throws IOException {
+ return cluster.getOzoneManager().getKeyManager()
+ .getKeyInfo(new OmKeyArgs.Builder().setVolumeName(volume)
+ .setBucketName(bucket).setKeyName(key).build(), null);
+ }
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Snapshot snap2 is created
+ * 4) Key k1 is deleted.
+ * 5) Snapdiff b/w snap3 & snap2 taken to assert difference of 1 key
+ */
+ @Test
+ public void testSnapDiffHandlingReclaimWithLatestUse() throws Exception {
+ String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5);
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ volume.createBucket(testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+ OmKeyInfo keyInfo = getOmKeyInfo(testVolumeName, testBucketName, key1);
+ bucket.deleteKey(key1);
+ String snap3 = "snap3";
+ String activeSnapshotPrefix =
+ createSnapshot(testVolumeName, testBucketName, snap3);
+ SnapshotDiffReportOzone diff =
+ getSnapDiffReport(testVolumeName, testBucketName, snap1, snap2);
+ Assert.assertEquals(diff.getDiffList().size(), 0);
+ diff = getSnapDiffReport(testVolumeName, testBucketName, snap2, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 1);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1)));
+ }
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Key k1 is deleted.
+ * 4) Snapshot snap2 is created.
+ * 5) Snapshot snap3 is created.
+ * 6) Snapdiff b/w snap3 & snap1 taken to assert difference of 1 key.
+ * 7) Snapdiff b/w snap3 & snap2 taken to assert difference of 0 key.
+ */
+ @Test
+ public void testSnapDiffHandlingReclaimWithPreviousUse() throws Exception {
+ String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5);
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ volume.createBucket(testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ OmKeyInfo keyInfo = getOmKeyInfo(testVolumeName, testBucketName, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ bucket.deleteKey(key1);
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+ String snap3 = "snap3";
+ String activeSnapshotPrefix = createSnapshot(testVolumeName, testBucketName,
+ snap3);
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
+ testBucketName, snap1, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 1);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1)));
+ diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap2, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 0);
+ }
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Key k1 is deleted.
+ * 4) Key k1 is recreated.
+ * 5) Snapshot snap2 is created.
+ * 6) Snapdiff b/w Active FS & snap1 taken to assert difference of 2 keys.
+ * 7) Snapdiff b/w Active FS & snap2 taken to assert difference of 0 key.
+ * 8) Checking rocks db to ensure the object created shouldn't be reclaimed
+ * as it is used by snapshot.
+ * 9) Key k1 is deleted.
+ * 10) Snapdiff b/w Active FS & snap1 taken to assert difference of 1 key.
+ * 11) Snapdiff b/w Active FS & snap2 taken to assert difference of 1 key.
+ */
+ @Test
+ public void testSnapDiffReclaimWithKeyRecreation() throws Exception {
+ String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5);
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ volume.createBucket(testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ OmKeyInfo keyInfo1 = getOmKeyInfo(testVolumeName, testBucketName, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ bucket.deleteKey(key1);
+ key1 = createFileKey(bucket, key1);
+ OmKeyInfo keyInfo2 = getOmKeyInfo(testVolumeName, testBucketName, key1);
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+ String snap3 = "snap3";
+ String activeSnapshotPrefix = createSnapshot(testVolumeName, testBucketName,
+ snap3);
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
+ testBucketName, snap1, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 2);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1),
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.CREATE, OZONE_URI_DELIMITER + key1)));
+ diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap2, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 0);
+ bucket.deleteKey(key1);
+ String snap4 = "snap4";
+ activeSnapshotPrefix = createSnapshot(testVolumeName, testBucketName,
+ snap4);
+ diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap1, snap4);
+ Assert.assertEquals(diff.getDiffList().size(), 1);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1)));
+ diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap2, snap4);
+ Assert.assertEquals(diff.getDiffList().size(), 1);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1)));
+ }
+
+
Review Comment:
nit: please remove extra line.
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java:
##########
@@ -547,6 +572,588 @@ public void testCreateSnapshotMissingMandatoryParams() throws Exception {
() -> createSnapshot(nullstr, bucket));
}
+ private Set<OmKeyInfo> getDeletedKeysFromRocksDb(
+ OMMetadataManager metadataManager) throws IOException {
+ Set<OmKeyInfo> deletedKeys = Sets.newHashSet();
+ try (TableIterator<String,
+ ? extends Table.KeyValue<String, RepeatedOmKeyInfo>>
+ deletedTableIterator = metadataManager.getDeletedTable()
+ .iterator()) {
+ while (deletedTableIterator.hasNext()) {
+ Table.KeyValue<String, RepeatedOmKeyInfo> val =
+ deletedTableIterator.next();
+ deletedKeys.addAll(val.getValue().getOmKeyInfoList());
+ }
+ }
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
+ deletedDirTableIterator = metadataManager.getDeletedDirTable()
+ .iterator()) {
+ while (deletedDirTableIterator.hasNext()) {
+ deletedKeys.add(deletedDirTableIterator.next().getValue());
+ }
+ }
+ return deletedKeys;
+ }
+
+ private OmKeyInfo getOmKeyInfo(String volume, String bucket,
+ String key) throws IOException {
+ return cluster.getOzoneManager().getKeyManager()
+ .getKeyInfo(new OmKeyArgs.Builder().setVolumeName(volume)
+ .setBucketName(bucket).setKeyName(key).build(), null);
+ }
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Snapshot snap2 is created
+ * 4) Key k1 is deleted.
+ * 5) Snapdiff b/w snap3 & snap2 taken to assert difference of 1 key
+ */
+ @Test
+ public void testSnapDiffHandlingReclaimWithLatestUse() throws Exception {
+ String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5);
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ volume.createBucket(testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+ OmKeyInfo keyInfo = getOmKeyInfo(testVolumeName, testBucketName, key1);
+ bucket.deleteKey(key1);
+ String snap3 = "snap3";
+ String activeSnapshotPrefix =
+ createSnapshot(testVolumeName, testBucketName, snap3);
+ SnapshotDiffReportOzone diff =
+ getSnapDiffReport(testVolumeName, testBucketName, snap1, snap2);
+ Assert.assertEquals(diff.getDiffList().size(), 0);
+ diff = getSnapDiffReport(testVolumeName, testBucketName, snap2, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 1);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1)));
+ }
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Key k1 is deleted.
+ * 4) Snapshot snap2 is created.
+ * 5) Snapshot snap3 is created.
+ * 6) Snapdiff b/w snap3 & snap1 taken to assert difference of 1 key.
+ * 7) Snapdiff b/w snap3 & snap2 taken to assert difference of 0 key.
+ */
+ @Test
+ public void testSnapDiffHandlingReclaimWithPreviousUse() throws Exception {
+ String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5);
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ volume.createBucket(testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ OmKeyInfo keyInfo = getOmKeyInfo(testVolumeName, testBucketName, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ bucket.deleteKey(key1);
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+ String snap3 = "snap3";
+ String activeSnapshotPrefix = createSnapshot(testVolumeName, testBucketName,
+ snap3);
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
+ testBucketName, snap1, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 1);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1)));
+ diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap2, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 0);
+ }
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Key k1 is deleted.
+ * 4) Key k1 is recreated.
+ * 5) Snapshot snap2 is created.
+ * 6) Snapdiff b/w Active FS & snap1 taken to assert difference of 2 keys.
Review Comment:
I don't think it is right wording because there is no way to get diff between Active FS or a snapshot. May be
```suggestion
* 6) Snapdiff b/w snapshot of Active FS & snap1 to assert difference of 2 keys.
```
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java:
##########
@@ -547,6 +572,588 @@ public void testCreateSnapshotMissingMandatoryParams() throws Exception {
() -> createSnapshot(nullstr, bucket));
}
+ private Set<OmKeyInfo> getDeletedKeysFromRocksDb(
+ OMMetadataManager metadataManager) throws IOException {
+ Set<OmKeyInfo> deletedKeys = Sets.newHashSet();
+ try (TableIterator<String,
+ ? extends Table.KeyValue<String, RepeatedOmKeyInfo>>
+ deletedTableIterator = metadataManager.getDeletedTable()
+ .iterator()) {
+ while (deletedTableIterator.hasNext()) {
+ Table.KeyValue<String, RepeatedOmKeyInfo> val =
+ deletedTableIterator.next();
+ deletedKeys.addAll(val.getValue().getOmKeyInfoList());
+ }
+ }
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
+ deletedDirTableIterator = metadataManager.getDeletedDirTable()
+ .iterator()) {
+ while (deletedDirTableIterator.hasNext()) {
+ deletedKeys.add(deletedDirTableIterator.next().getValue());
+ }
+ }
+ return deletedKeys;
+ }
+
+ private OmKeyInfo getOmKeyInfo(String volume, String bucket,
+ String key) throws IOException {
+ return cluster.getOzoneManager().getKeyManager()
+ .getKeyInfo(new OmKeyArgs.Builder().setVolumeName(volume)
+ .setBucketName(bucket).setKeyName(key).build(), null);
+ }
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Snapshot snap2 is created
+ * 4) Key k1 is deleted.
+ * 5) Snapdiff b/w snap3 & snap2 taken to assert difference of 1 key
+ */
+ @Test
+ public void testSnapDiffHandlingReclaimWithLatestUse() throws Exception {
+ String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5);
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ volume.createBucket(testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+ OmKeyInfo keyInfo = getOmKeyInfo(testVolumeName, testBucketName, key1);
+ bucket.deleteKey(key1);
+ String snap3 = "snap3";
+ String activeSnapshotPrefix =
+ createSnapshot(testVolumeName, testBucketName, snap3);
+ SnapshotDiffReportOzone diff =
+ getSnapDiffReport(testVolumeName, testBucketName, snap1, snap2);
+ Assert.assertEquals(diff.getDiffList().size(), 0);
+ diff = getSnapDiffReport(testVolumeName, testBucketName, snap2, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 1);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1)));
+ }
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Key k1 is deleted.
+ * 4) Snapshot snap2 is created.
+ * 5) Snapshot snap3 is created.
+ * 6) Snapdiff b/w snap3 & snap1 taken to assert difference of 1 key.
+ * 7) Snapdiff b/w snap3 & snap2 taken to assert difference of 0 key.
+ */
+ @Test
+ public void testSnapDiffHandlingReclaimWithPreviousUse() throws Exception {
+ String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5);
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ volume.createBucket(testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ OmKeyInfo keyInfo = getOmKeyInfo(testVolumeName, testBucketName, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ bucket.deleteKey(key1);
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+ String snap3 = "snap3";
+ String activeSnapshotPrefix = createSnapshot(testVolumeName, testBucketName,
+ snap3);
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
+ testBucketName, snap1, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 1);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1)));
+ diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap2, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 0);
+ }
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Key k1 is deleted.
+ * 4) Key k1 is recreated.
+ * 5) Snapshot snap2 is created.
+ * 6) Snapdiff b/w Active FS & snap1 taken to assert difference of 2 keys.
+ * 7) Snapdiff b/w Active FS & snap2 taken to assert difference of 0 key.
+ * 8) Checking rocks db to ensure the object created shouldn't be reclaimed
+ * as it is used by snapshot.
+ * 9) Key k1 is deleted.
+ * 10) Snapdiff b/w Active FS & snap1 taken to assert difference of 1 key.
+ * 11) Snapdiff b/w Active FS & snap2 taken to assert difference of 1 key.
+ */
+ @Test
+ public void testSnapDiffReclaimWithKeyRecreation() throws Exception {
+ String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5);
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ volume.createBucket(testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ OmKeyInfo keyInfo1 = getOmKeyInfo(testVolumeName, testBucketName, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ bucket.deleteKey(key1);
+ key1 = createFileKey(bucket, key1);
+ OmKeyInfo keyInfo2 = getOmKeyInfo(testVolumeName, testBucketName, key1);
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+ String snap3 = "snap3";
+ String activeSnapshotPrefix = createSnapshot(testVolumeName, testBucketName,
+ snap3);
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
+ testBucketName, snap1, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 2);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1),
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.CREATE, OZONE_URI_DELIMITER + key1)));
+ diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap2, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 0);
+ bucket.deleteKey(key1);
+ String snap4 = "snap4";
+ activeSnapshotPrefix = createSnapshot(testVolumeName, testBucketName,
+ snap4);
+ diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap1, snap4);
+ Assert.assertEquals(diff.getDiffList().size(), 1);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1)));
+ diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap2, snap4);
+ Assert.assertEquals(diff.getDiffList().size(), 1);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1)));
+ }
+
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Key k1 is renamed to renamed-k1.
+ * 4) Key renamed-k1 is deleted.
+ * 5) Snapshot snap2 created.
+ * 4) Snapdiff b/w snap2 & snap1 taken to assert difference of 1 key.
+ */
+ @Test
+ public void testSnapDiffReclaimWithKeyRename() throws Exception {
+ String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5);
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ volume.createBucket(testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ String renamedKey = "renamed-" + key1;
+ bucket.renameKey(key1, renamedKey);
+ GenericTestUtils.waitFor(
+ () -> {
+ try {
+ getOmKeyInfo(testVolumeName, testBucketName, renamedKey);
+ } catch (IOException e) {
+ return false;
+ }
+ return true;
+ }, 1000, 10000);
+ OmKeyInfo renamedKeyInfo = getOmKeyInfo(testVolumeName, testBucketName,
+ renamedKey);
+ bucket.deleteKey(renamedKey);
+ String snap2 = "snap2";
+ String activeSnapshotPrefix = createSnapshot(testVolumeName, testBucketName,
+ snap2);
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
+ testBucketName, snap1, snap2);
+ Assert.assertEquals(diff.getDiffList().size(), 1);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1)
+ ));
+ }
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Key k1 is renamed to renamed-k1.
+ * 4) Key renamed-k1 is renamed to renamed-renamed-k1.
+ * 5) Key renamed-renamed-k1 is deleted.
+ * 6) Snapshot snap2 is created.
+ * 7) Snapdiff b/w Active FS & snap1 taken to assert difference of 1 key.
+ */
+ @Test
+ public void testSnapDiffWith2RenamesAndDelete() throws Exception {
+ String testVolumeName = "vol" + counter.incrementAndGet();
+ String testBucketName = "bucket" + counter.incrementAndGet();
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ volume.createBucket(testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ String renamedKey = "renamed-" + key1;
+ bucket.renameKey(key1, renamedKey);
+ GenericTestUtils.waitFor(
+ () -> {
+ try {
+ getOmKeyInfo(testVolumeName, testBucketName, renamedKey);
+ } catch (IOException e) {
+ return false;
+ }
+ return true;
+ }, 1000, 120000);
+ String renamedRenamedKey = "renamed-" + renamedKey;
+ bucket.renameKey(renamedKey, renamedRenamedKey);
+ GenericTestUtils.waitFor(
Review Comment:
Same as above.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org