You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2023/04/15 17:50:40 UTC
[ozone] branch master updated: HDDS-8370. Intermittent failure in TestOzoneFileSystem#testListStatusWithIntermediateDir (#4564)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 617ef338cd HDDS-8370. Intermittent failure in TestOzoneFileSystem#testListStatusWithIntermediateDir (#4564)
617ef338cd is described below
commit 617ef338cd5e0e02c8ee8b66be07af3e751ff787
Author: Doroszlai, Attila <64...@users.noreply.github.com>
AuthorDate: Sat Apr 15 19:50:35 2023 +0200
HDDS-8370. Intermittent failure in TestOzoneFileSystem#testListStatusWithIntermediateDir (#4564)
---
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 84 +++++++++++-----------
1 file changed, 40 insertions(+), 44 deletions(-)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
index 547253dd3b..b4dacb1aeb 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.Trash;
@@ -112,6 +113,15 @@ public class TestOzoneFileSystem {
private static final float TRASH_INTERVAL = 0.05f; // 3 seconds
+ private static final Path ROOT =
+ new Path(OZONE_URI_DELIMITER);
+
+ private static final Path TRASH_ROOT =
+ new Path(ROOT, TRASH_PREFIX);
+
+ private static final PathFilter EXCLUDE_TRASH =
+ p -> !p.toUri().getPath().startsWith(TRASH_ROOT.toString());
+
@Parameterized.Parameters
public static Collection<Object[]> data() {
return Arrays.asList(
@@ -215,8 +225,7 @@ public class TestOzoneFileSystem {
@After
public void cleanup() {
try {
- Path root = new Path("/");
- FileStatus[] fileStatuses = fs.listStatus(root);
+ FileStatus[] fileStatuses = fs.listStatus(ROOT);
for (FileStatus fileStatus : fileStatuses) {
fs.delete(fileStatus.getPath(), true);
}
@@ -576,18 +585,17 @@ public class TestOzoneFileSystem {
@Test
public void testListStatus() throws Exception {
- Path root = new Path("/");
- Path parent = new Path(root, "/testListStatus");
+ Path parent = new Path(ROOT, "/testListStatus");
Path file1 = new Path(parent, "key1");
Path file2 = new Path(parent, "key2");
- FileStatus[] fileStatuses = o3fs.listStatus(root);
+ FileStatus[] fileStatuses = o3fs.listStatus(ROOT, EXCLUDE_TRASH);
Assert.assertEquals("Should be empty", 0, fileStatuses.length);
ContractTestUtils.touch(fs, file1);
ContractTestUtils.touch(fs, file2);
- fileStatuses = o3fs.listStatus(root);
+ fileStatuses = o3fs.listStatus(ROOT, EXCLUDE_TRASH);
Assert.assertEquals("Should have created parent",
1, fileStatuses.length);
Assert.assertEquals("Parent path doesn't match",
@@ -624,13 +632,11 @@ public class TestOzoneFileSystem {
OpenKeySession session = writeClient.openKey(keyArgs);
writeClient.commitKey(keyArgs, session.getId());
- Path parent = new Path("/");
-
// Wait until the filestatus is updated
if (!enabledFileSystemPaths) {
GenericTestUtils.waitFor(() -> {
try {
- return fs.listStatus(parent).length != 0;
+ return fs.listStatus(ROOT, EXCLUDE_TRASH).length != 0;
} catch (IOException e) {
LOG.error("listStatus() Failed", e);
Assert.fail("listStatus() Failed");
@@ -639,10 +645,10 @@ public class TestOzoneFileSystem {
}, 1000, 120000);
}
- FileStatus[] fileStatuses = fs.listStatus(parent);
+ FileStatus[] fileStatuses = fs.listStatus(ROOT, EXCLUDE_TRASH);
// the number of immediate children of root is 1
- Assert.assertEquals(1, fileStatuses.length);
+ Assert.assertEquals(Arrays.toString(fileStatuses), 1, fileStatuses.length);
writeClient.deleteKey(keyArgs);
}
@@ -660,12 +666,11 @@ public class TestOzoneFileSystem {
.build();
OpenKeySession session = writeClient.openKey(keyArgs);
writeClient.commitKey(keyArgs, session.getId());
- Path parent = new Path("/");
// Wait until the filestatus is updated
if (!enabledFileSystemPaths) {
GenericTestUtils.waitFor(() -> {
try {
- return fs.listStatus(parent).length != 0;
+ return fs.listStatus(ROOT, EXCLUDE_TRASH).length != 0;
} catch (IOException e) {
LOG.error("listStatus() Failed", e);
Assert.fail("listStatus() Failed");
@@ -673,7 +678,7 @@ public class TestOzoneFileSystem {
}
}, 1000, 120000);
}
- FileStatus[] fileStatuses = fs.listStatus(parent);
+ FileStatus[] fileStatuses = fs.listStatus(ROOT, EXCLUDE_TRASH);
// the number of immediate children of root is 1
Assert.assertEquals(1, fileStatuses.length);
Assert.assertEquals(fileStatuses[0].isErasureCoded(),
@@ -690,17 +695,16 @@ public class TestOzoneFileSystem {
*/
@Test
public void testListStatusOnRoot() throws Exception {
- Path root = new Path("/");
- Path dir1 = new Path(root, "dir1");
+ Path dir1 = new Path(ROOT, "dir1");
Path dir12 = new Path(dir1, "dir12");
- Path dir2 = new Path(root, "dir2");
+ Path dir2 = new Path(ROOT, "dir2");
fs.mkdirs(dir12);
fs.mkdirs(dir2);
// ListStatus on root should return dir1 (even though /dir1 key does not
// exist) and dir2 only. dir12 is not an immediate child of root and
// hence should not be listed.
- FileStatus[] fileStatuses = o3fs.listStatus(root);
+ FileStatus[] fileStatuses = o3fs.listStatus(ROOT, EXCLUDE_TRASH);
assertEquals("FileStatus should return only the immediate children",
2, fileStatuses.length);
@@ -716,17 +720,16 @@ public class TestOzoneFileSystem {
*/
@Test
public void testListStatusOnLargeDirectory() throws Exception {
- Path root = new Path("/");
deleteRootDir(); // cleanup
Set<String> paths = new TreeSet<>();
int numDirs = LISTING_PAGE_SIZE + LISTING_PAGE_SIZE / 2;
for (int i = 0; i < numDirs; i++) {
- Path p = new Path(root, String.valueOf(i));
+ Path p = new Path(ROOT, String.valueOf(i));
fs.mkdirs(p);
paths.add(p.getName());
}
- FileStatus[] fileStatuses = o3fs.listStatus(root);
+ FileStatus[] fileStatuses = o3fs.listStatus(ROOT, EXCLUDE_TRASH);
// Added logs for debugging failures, to check any sub-path mismatches.
Set<String> actualPaths = new TreeSet<>();
ArrayList<String> actualPathList = new ArrayList<>();
@@ -772,7 +775,7 @@ public class TestOzoneFileSystem {
TestDataUtil.createKey(ozoneBucket, keyName, "");
FileStatus[] fileStatuses;
- fileStatuses = fs.listStatus(new Path("/"));
+ fileStatuses = fs.listStatus(ROOT, EXCLUDE_TRASH);
assertEquals(1, fileStatuses.length);
assertEquals("/dir1", fileStatuses[0].getPath().toUri().getPath());
assertTrue(fileStatuses[0].isDirectory());
@@ -795,8 +798,7 @@ public class TestOzoneFileSystem {
* @throws IOException DB failure
*/
protected void deleteRootDir() throws IOException {
- Path root = new Path("/");
- FileStatus[] fileStatuses = fs.listStatus(root);
+ FileStatus[] fileStatuses = fs.listStatus(ROOT);
if (fileStatuses == null) {
return;
@@ -806,7 +808,7 @@ public class TestOzoneFileSystem {
fs.delete(fStatus.getPath(), true);
}
- fileStatuses = fs.listStatus(root);
+ fileStatuses = fs.listStatus(ROOT);
if (fileStatuses != null) {
Assert.assertEquals("Delete root failed!", 0, fileStatuses.length);
}
@@ -856,19 +858,18 @@ public class TestOzoneFileSystem {
*/
@Test
public void testListStatusIteratorWithDir() throws Exception {
- Path root = new Path("/");
- Path parent = new Path(root, "testListStatus");
+ Path parent = new Path(ROOT, "testListStatus");
Path file1 = new Path(parent, "key1");
Path file2 = new Path(parent, "key2");
try {
// Iterator should have no items when dir is empty
- RemoteIterator<FileStatus> it = o3fs.listStatusIterator(root);
+ RemoteIterator<FileStatus> it = o3fs.listStatusIterator(ROOT);
Assert.assertFalse(it.hasNext());
ContractTestUtils.touch(fs, file1);
ContractTestUtils.touch(fs, file2);
// Iterator should have an item when dir is not empty
- it = o3fs.listStatusIterator(root);
+ it = o3fs.listStatusIterator(ROOT);
while (it.hasNext()) {
FileStatus fileStatus = it.next();
Assert.assertNotNull(fileStatus);
@@ -915,10 +916,9 @@ public class TestOzoneFileSystem {
*/
@Test
public void testListStatusIteratorOnRoot() throws Exception {
- Path root = new Path("/");
- Path dir1 = new Path(root, "dir1");
+ Path dir1 = new Path(ROOT, "dir1");
Path dir12 = new Path(dir1, "dir12");
- Path dir2 = new Path(root, "dir2");
+ Path dir2 = new Path(ROOT, "dir2");
try {
fs.mkdirs(dir12);
fs.mkdirs(dir2);
@@ -926,7 +926,7 @@ public class TestOzoneFileSystem {
// ListStatusIterator on root should return dir1
// (even though /dir1 key does not exist)and dir2 only.
// dir12 is not an immediate child of root and hence should not be listed.
- RemoteIterator<FileStatus> it = o3fs.listStatusIterator(root);
+ RemoteIterator<FileStatus> it = o3fs.listStatusIterator(ROOT);
int iCount = 0;
while (it.hasNext()) {
iCount++;
@@ -1110,7 +1110,7 @@ public class TestOzoneFileSystem {
public void testDeleteRoot() throws IOException {
Path dir = new Path("/dir");
fs.mkdirs(dir);
- assertFalse(fs.delete(new Path("/"), true));
+ assertFalse(fs.delete(ROOT, true));
assertNotNull(fs.getFileStatus(dir));
}
@@ -1505,12 +1505,11 @@ public class TestOzoneFileSystem {
@Test
public void testGetTrashRoot() throws IOException {
String username = UserGroupInformation.getCurrentUser().getShortUserName();
- Path trashRoot = new Path(OZONE_URI_DELIMITER, TRASH_PREFIX);
// Input path doesn't matter, o3fs.getTrashRoot() only cares about username
Path inPath1 = new Path("o3fs://bucket2.volume1/path/to/key");
// Test with current user
Path outPath1 = o3fs.getTrashRoot(inPath1);
- Path expectedOutPath1 = new Path(trashRoot, username);
+ Path expectedOutPath1 = new Path(TRASH_ROOT, username);
Assert.assertEquals(expectedOutPath1, outPath1);
}
@@ -1577,8 +1576,7 @@ public class TestOzoneFileSystem {
@Test
public void testGetTrashRoots() throws IOException {
String username = UserGroupInformation.getCurrentUser().getShortUserName();
- Path trashRoot = new Path(OZONE_URI_DELIMITER, TRASH_PREFIX);
- Path userTrash = new Path(trashRoot, username);
+ Path userTrash = new Path(TRASH_ROOT, username);
Collection<FileStatus> res = o3fs.getTrashRoots(false);
Assert.assertEquals(0, res.size());
@@ -1594,12 +1592,12 @@ public class TestOzoneFileSystem {
// Create a few more random user trash dir
for (int i = 1; i <= 5; i++) {
- Path moreUserTrash = new Path(trashRoot, "trashuser" + i);
+ Path moreUserTrash = new Path(TRASH_ROOT, "trashuser" + i);
fs.mkdirs(moreUserTrash);
}
// And create a file, which should be ignored
- fs.create(new Path(trashRoot, "trashuser99"));
+ fs.create(new Path(TRASH_ROOT, "trashuser99"));
// allUsers = false should still return current user trash
res = o3fs.getTrashRoots(false);
@@ -1630,8 +1628,7 @@ public class TestOzoneFileSystem {
// Construct paths
String username = UserGroupInformation.getCurrentUser().getShortUserName();
- Path trashRoot = new Path(OZONE_URI_DELIMITER, TRASH_PREFIX);
- Path userTrash = new Path(trashRoot, username);
+ Path userTrash = new Path(TRASH_ROOT, username);
Path userTrashCurrent = new Path(userTrash, "Current");
Path trashPath = new Path(userTrashCurrent, testKeyName);
@@ -1661,8 +1658,7 @@ public class TestOzoneFileSystem {
// Construct paths
String username = UserGroupInformation.getCurrentUser().getShortUserName();
- Path trashRoot = new Path(OZONE_URI_DELIMITER, TRASH_PREFIX);
- Path userTrash = new Path(trashRoot, username);
+ Path userTrash = new Path(TRASH_ROOT, username);
Path userTrashCurrent = new Path(userTrash, "Current");
Path trashPath = new Path(userTrashCurrent, testKeyName);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org