You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ms...@apache.org on 2021/02/16 08:58:10 UTC
[ozone] branch master updated: HDDS-4825. Trash checkpoints are not
getting deleted from multiple buckets. (#1922)
This is an automated email from the ASF dual-hosted git repository.
msingh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 6e6bd25 HDDS-4825. Trash checkpoints are not getting deleted from multiple buckets. (#1922)
6e6bd25 is described below
commit 6e6bd256c9826515ac5dd182b2bdd3bf5efc624a
Author: Sadanand Shenoy <sa...@gmail.com>
AuthorDate: Tue Feb 16 14:27:53 2021 +0530
HDDS-4825. Trash checkpoints are not getting deleted from multiple buckets. (#1922)
---
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 55 ++++++++++++++++++----
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 12 ++---
2 files changed, 52 insertions(+), 15 deletions(-)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
index a3f56a9..710c906 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
@@ -1195,15 +1195,27 @@ public class TestRootedOzoneFileSystem {
/**
* 1.Move a Key to Trash
* 2.Verify that the key gets deleted by the trash emptier.
+ * 3.Create a second Key in different bucket and verify deletion.
* @throws Exception
*/
@Test
public void testTrash() throws Exception {
String testKeyName = "keyToBeDeleted";
- Path path = new Path(bucketPath, testKeyName);
- try (FSDataOutputStream stream = fs.create(path)) {
+ Path keyPath1 = new Path(bucketPath, testKeyName);
+ try (FSDataOutputStream stream = fs.create(keyPath1)) {
+ stream.write(1);
+ }
+ // create second bucket and write a key in it.
+ OzoneBucket bucket2 = TestDataUtil.createVolumeAndBucket(cluster);
+ String volumeName2 = bucket2.getVolumeName();
+ Path volumePath2 = new Path(OZONE_URI_DELIMITER, volumeName2);
+ String bucketName2 = bucket2.getName();
+ Path bucketPath2 = new Path(volumePath2, bucketName2);
+ Path keyPath2 = new Path(bucketPath2, testKeyName + "1");
+ try (FSDataOutputStream stream = fs.create(keyPath2)) {
stream.write(1);
}
+
Assert.assertTrue(trash.getConf().getClass(
"fs.trash.classname", TrashPolicy.class).
isAssignableFrom(TrashPolicyOzone.class));
@@ -1215,20 +1227,26 @@ public class TestRootedOzoneFileSystem {
long prevNumTrashFileRenames = getOMMetrics().getNumTrashFilesRenames();
// Call moveToTrash. We can't call protected fs.rename() directly
- trash.moveToTrash(path);
+ trash.moveToTrash(keyPath1);
+ // for key in second bucket
+ trash.moveToTrash(keyPath2);
- // Construct paths
+ // Construct paths for first key
String username = UserGroupInformation.getCurrentUser().getShortUserName();
Path trashRoot = new Path(bucketPath, TRASH_PREFIX);
Path userTrash = new Path(trashRoot, username);
- Path userTrashCurrent = new Path(userTrash, "Current");
- String key = path.toString().substring(1);
- Path trashPath = new Path(userTrashCurrent, key);
+ Path trashPath = getTrashKeyPath(keyPath1, userTrash);
+
+ // Construct paths for second key in different bucket
+ Path trashRoot2 = new Path(bucketPath2, TRASH_PREFIX);
+ Path userTrash2 = new Path(trashRoot2, username);
+ Path trashPath2 = getTrashKeyPath(keyPath2, userTrash2);
- // Wait until the TrashEmptier purges the key
+
+ // Wait until the TrashEmptier purges the keys
GenericTestUtils.waitFor(()-> {
try {
- return !ofs.exists(trashPath);
+ return !ofs.exists(trashPath) && !ofs.exists(trashPath2);
} catch (IOException e) {
LOG.error("Delete from Trash Failed", e);
Assert.fail("Delete from Trash Failed");
@@ -1242,6 +1260,18 @@ public class TestRootedOzoneFileSystem {
Assert.assertTrue(getOMMetrics()
.getNumTrashFilesRenames() > prevNumTrashFileRenames);
+ // wait for deletion of checkpoint dir
+ GenericTestUtils.waitFor(()-> {
+ try {
+ return ofs.listStatus(userTrash).length==0 &&
+ ofs.listStatus(userTrash2).length==0;
+ } catch (IOException e) {
+ LOG.error("Delete from Trash Failed", e);
+ Assert.fail("Delete from Trash Failed");
+ return false;
+ }
+ }, 1000, 120000);
+
// This condition should succeed once the checkpoint directory is deleted
GenericTestUtils.waitFor(
() -> getOMMetrics().getNumTrashDeletes() > prevNumTrashDeletes
@@ -1249,9 +1279,16 @@ public class TestRootedOzoneFileSystem {
> prevNumTrashFileDeletes, 100, 180000);
// Cleanup
ofs.delete(trashRoot, true);
+ ofs.delete(trashRoot2, true);
}
+ private Path getTrashKeyPath(Path keyPath, Path userTrash) {
+ Path userTrashCurrent = new Path(userTrash, "Current");
+ String key = keyPath.toString().substring(1);
+ return new Path(userTrashCurrent, key);
+ }
+
@Test
public void testCreateWithInvalidPaths() throws Exception {
// Test for path with ..
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java
index d7606d2..6d7a88a 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java
@@ -75,8 +75,6 @@ public class TrashOzoneFileSystem extends FileSystem {
private final String userName;
- private String ofsPathPrefix;
-
private final AtomicLong runCount;
private static final ClientId CLIENT_ID = ClientId.randomId();
@@ -192,8 +190,12 @@ public class TrashOzoneFileSystem extends FileSystem {
* converts OzoneFileStatus object to FileStatus.
*/
private FileStatus convertToFileStatus(OzoneFileStatus status) {
- Path temp = new Path(ofsPathPrefix +
- OZONE_URI_DELIMITER + status.getKeyInfo().getKeyName());
+ Path temp = new Path(OZONE_URI_DELIMITER +
+ status.getKeyInfo().getVolumeName() +
+ OZONE_URI_DELIMITER +
+ status.getKeyInfo().getBucketName() +
+ OZONE_URI_DELIMITER +
+ status.getKeyInfo().getKeyName());
return new FileStatus(
status.getKeyInfo().getDataSize(),
status.isDirectory(),
@@ -243,8 +245,6 @@ public class TrashOzoneFileSystem extends FileSystem {
.setBucketName(bucket)
.setKeyName(key)
.build();
- this.ofsPathPrefix = OZONE_URI_DELIMITER +
- volume + OZONE_URI_DELIMITER + bucket;
return keyArgs;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org