You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by sa...@apache.org on 2022/09/25 10:47:48 UTC
[ozone] branch master updated: HDDS-7075. Implement FileSystem listStatusIterator to support file list paging (#3711)
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new f6f89a021c HDDS-7075. Implement FileSystem listStatusIterator to support file list paging (#3711)
f6f89a021c is described below
commit f6f89a021cef6ee60cb5940cab20d681bdc90e5b
Author: Navink <nv...@gmail.com>
AuthorDate: Sun Sep 25 16:17:42 2022 +0530
HDDS-7075. Implement FileSystem listStatusIterator to support file list paging (#3711)
---
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 203 +++++++++++++++++++
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 218 +++++++++++++++++++++
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 127 ++++++++++++
.../fs/ozone/BasicRootedOzoneFileSystem.java | 127 ++++++++++++
4 files changed, 675 insertions(+)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
index 9bd8e86b7b..3630d7a5da 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.fs.TrashPolicy;
import org.apache.hadoop.fs.contract.ContractTestUtils;
@@ -751,6 +752,208 @@ public class TestOzoneFileSystem {
fileStatus2.equals(dir12.toString()));
}
+ /**
+ * Tests listStatusIterator operation on root directory.
+ */
+ @Test
+ public void testListStatusIteratorWithDir() throws Exception {
+ Path root = new Path("/");
+ Path parent = new Path(root, "testListStatus");
+ Path file1 = new Path(parent, "key1");
+ Path file2 = new Path(parent, "key2");
+ try {
+ // Iterator should have no items when dir is empty
+ RemoteIterator<FileStatus> it = o3fs.listStatusIterator(root);
+ Assert.assertFalse(it.hasNext());
+
+ ContractTestUtils.touch(fs, file1);
+ ContractTestUtils.touch(fs, file2);
+ // Iterator should have an item when dir is not empty
+ it = o3fs.listStatusIterator(root);
+ while (it.hasNext()) {
+ FileStatus fileStatus = it.next();
+ Assert.assertNotNull(fileStatus);
+ Assert.assertEquals("Parent path doesn't match",
+ fileStatus.getPath().toUri().getPath(), parent.toString());
+ }
+ // Iterator on a directory should return all subdirs along with
+ // files, even if there exists a file and sub-dir with the same name.
+ it = o3fs.listStatusIterator(parent);
+ int iCount = 0;
+ while (it.hasNext()) {
+ iCount++;
+ FileStatus fileStatus = it.next();
+ Assert.assertNotNull(fileStatus);
+ }
+ Assert.assertEquals(
+ "Iterator did not return all the file status",
+ 2, iCount);
+ // Iterator should return file status for only the
+ // immediate children of a directory.
+ Path file3 = new Path(parent, "dir1/key3");
+ Path file4 = new Path(parent, "dir1/key4");
+ ContractTestUtils.touch(fs, file3);
+ ContractTestUtils.touch(fs, file4);
+ it = o3fs.listStatusIterator(parent);
+ iCount = 0;
+
+ while (it.hasNext()) {
+ iCount++;
+ FileStatus fileStatus = it.next();
+ Assert.assertNotNull(fileStatus);
+ }
+ Assert.assertEquals("Iterator did not return file status " +
+ "of all the children of the directory", 3, iCount);
+
+ } finally {
+ // Cleanup
+ fs.delete(parent, true);
+ }
+ }
+
+ /**
+ * Tests listStatusIterator operation on root directory.
+ */
+ @Test
+ public void testListStatusIteratorOnRoot() throws Exception {
+ Path root = new Path("/");
+ Path dir1 = new Path(root, "dir1");
+ Path dir12 = new Path(dir1, "dir12");
+ Path dir2 = new Path(root, "dir2");
+ try {
+ fs.mkdirs(dir12);
+ fs.mkdirs(dir2);
+
+ // ListStatusIterator on root should return dir1
+ // (even though /dir1 key does not exist)and dir2 only.
+ // dir12 is not an immediate child of root and hence should not be listed.
+ RemoteIterator<FileStatus> it = o3fs.listStatusIterator(root);
+ int iCount = 0;
+ while (it.hasNext()) {
+ iCount++;
+ FileStatus fileStatus = it.next();
+ Assert.assertNotNull(fileStatus);
+ // Verify that dir12 is not included in the result
+ // of the listStatusIterator on root.
+ assertNotEquals(fileStatus.getPath().toUri().getPath(),
+ dir12.toString());
+ }
+ assertEquals("FileStatus should return only the immediate children",
+ 2, iCount);
+ } finally {
+ // Cleanup
+ fs.delete(dir2, true);
+ fs.delete(dir1, true);
+ }
+ }
+
+ /**
+ * Tests listStatusIterator operation on root directory with different
+ * numbers of numDir.
+ */
+ @Test
+ public void testListStatusIteratorOnPageSize() throws Exception {
+ int[] pageSize = {
+ 1, LISTING_PAGE_SIZE, LISTING_PAGE_SIZE + 1,
+ LISTING_PAGE_SIZE - 1, LISTING_PAGE_SIZE + LISTING_PAGE_SIZE / 2,
+ LISTING_PAGE_SIZE + LISTING_PAGE_SIZE
+ };
+ for (int numDir : pageSize) {
+ int range = numDir / LISTING_PAGE_SIZE;
+ switch (range) {
+ case 0:
+ listStatusIterator(numDir);
+ break;
+ case 1:
+ listStatusIterator(numDir);
+ break;
+ case 2:
+ listStatusIterator(numDir);
+ break;
+ default:
+ listStatusIterator(numDir);
+ }
+ }
+ }
+
+ private void listStatusIterator(int numDirs) throws IOException {
+ Path root = new Path("/" + volumeName + "/" + bucketName);
+ Set<String> paths = new TreeSet<>();
+ try {
+ for (int i = 0; i < numDirs; i++) {
+ Path p = new Path(root, String.valueOf(i));
+ fs.mkdirs(p);
+ paths.add(p.getName());
+ }
+
+ RemoteIterator<FileStatus> iterator = o3fs.listStatusIterator(root);
+ int iCount = 0;
+ if (iterator != null) {
+ while (iterator.hasNext()) {
+ FileStatus fileStatus = iterator.next();
+ iCount++;
+ Assert.assertTrue(paths.contains(fileStatus.getPath().getName()));
+ }
+ }
+ Assert.assertEquals(
+ "Total directories listed do not match the existing directories",
+ numDirs, iCount);
+
+ } finally {
+ // Cleanup
+ for (int i = 0; i < numDirs; i++) {
+ Path p = new Path(root, String.valueOf(i));
+ fs.delete(p, true);
+ }
+ }
+ }
+
+ /**
+ * Tests listStatus on a path with subdirs.
+ */
+ @Test
+ public void testListStatusIteratorOnSubDirs() throws Exception {
+ // Create the following key structure
+ // /dir1/dir11/dir111
+ // /dir1/dir12
+ // /dir1/dir12/file121
+ // /dir2
+ // ListStatusIterator on /dir1 should return file status for
+ // all its immediate subdirs only which are /dir1/dir11 and
+ // /dir1/dir12. Super child files/dirs (/dir1/dir12/file121
+ // and /dir1/dir11/dir111) should not be returned by
+ // listStatusIterator.
+ Path dir1 = new Path("/dir1");
+ Path dir11 = new Path(dir1, "dir11");
+ Path dir111 = new Path(dir11, "dir111");
+ Path dir12 = new Path(dir1, "dir12");
+ Path file121 = new Path(dir12, "file121");
+ Path dir2 = new Path("/dir2");
+ try {
+ fs.mkdirs(dir111);
+ fs.mkdirs(dir12);
+ ContractTestUtils.touch(fs, file121);
+ fs.mkdirs(dir2);
+
+ RemoteIterator<FileStatus> it = o3fs.listStatusIterator(dir1);
+ int iCount = 0;
+ while (it.hasNext()) {
+ iCount++;
+ FileStatus fileStatus = it.next();
+ Assert.assertNotNull(fileStatus);
+ assertTrue(fileStatus.getPath().toUri().getPath().
+ equals(dir11.toString()) || fileStatus.getPath().toUri().getPath()
+ .equals(dir12.toString()));
+ }
+ assertEquals("FileStatus should return only the immediate children", 2,
+ iCount);
+ } finally {
+ // Cleanup
+ fs.delete(dir2, true);
+ fs.delete(dir1, true);
+ }
+ }
+
@Test
public void testSeekOnFileLength() throws IOException {
Path file = new Path("/file");
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
index da86169282..96ac47f065 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.StreamCapabilities;
import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.fs.TrashPolicy;
@@ -387,6 +388,223 @@ public class TestRootedOzoneFileSystem {
fs.delete(parent, true);
}
+ /**
+ * Tests listStatusIterator operation on a directory.
+ */
+ @Test
+ public void testListStatusIteratorWithDir() throws Exception {
+ Path parent = new Path(bucketPath, "testListStatus");
+ Path file1 = new Path(parent, "key1");
+ Path file2 = new Path(parent, "key2");
+ try {
+ // Iterator should have no items when dir is empty
+ RemoteIterator<FileStatus> it = ofs.listStatusIterator(bucketPath);
+ Assert.assertFalse(it.hasNext());
+ ContractTestUtils.touch(fs, file1);
+ ContractTestUtils.touch(fs, file2);
+ // Iterator should have an item when dir is not empty
+ it = ofs.listStatusIterator(bucketPath);
+ while (it.hasNext()) {
+ FileStatus fileStatus = it.next();
+ Assert.assertNotNull(fileStatus);
+ Assert.assertEquals("Parent path doesn't match",
+ fileStatus.getPath().toUri().getPath(), parent.toString());
+ }
+ // Iterator on a directory should return all subdirs along with
+ // files.
+ it = ofs.listStatusIterator(parent);
+ int iCount = 0;
+ while (it.hasNext()) {
+ iCount++;
+ FileStatus fileStatus = it.next();
+ Assert.assertNotNull(fileStatus);
+ }
+ Assert.assertEquals(
+ "Iterator did not return all the file status",
+ 2, iCount);
+ // Iterator should return file status for only the
+ // immediate children of a directory.
+ Path file3 = new Path(parent, "dir1/key3");
+ Path file4 = new Path(parent, "dir1/key4");
+ ContractTestUtils.touch(fs, file3);
+ ContractTestUtils.touch(fs, file4);
+ it = ofs.listStatusIterator(parent);
+ iCount = 0;
+ while (it.hasNext()) {
+ iCount++;
+ FileStatus fileStatus = it.next();
+ Assert.assertNotNull(fileStatus);
+ }
+ Assert.assertEquals("Iterator did not return file status " +
+ "of all the children of the directory", 3, iCount);
+ } finally {
+ // Cleanup
+ fs.delete(parent, true);
+ }
+ }
+
+ /**
+ * Test listStatusIterator operation in a bucket.
+ */
+ @Test
+ public void testListStatusIteratorInBucket() throws Exception {
+ Path root = new Path("/" + volumeName + "/" + bucketName);
+ Path dir1 = new Path(root, "dir1");
+ Path dir12 = new Path(dir1, "dir12");
+ Path dir2 = new Path(root, "dir2");
+ try {
+ fs.mkdirs(dir12);
+ fs.mkdirs(dir2);
+
+ // ListStatus on root should return dir1 (even though /dir1 key does not
+ // exist) and dir2 only. dir12 is not an immediate child of root and
+ // hence should not be listed.
+ RemoteIterator<FileStatus> it = ofs.listStatusIterator(root);
+ // Verify that dir12 is not included in the result of the listStatus on
+ // root
+ int iCount = 0;
+ while (it.hasNext()) {
+ iCount++;
+ FileStatus fileStatus = it.next();
+ Assert.assertNotNull(fileStatus);
+ Assert.assertNotEquals(fileStatus, dir12.toString());
+ }
+ Assert.assertEquals(
+ "FileStatus should return only the immediate children",
+ 2, iCount);
+
+ } finally {
+ // cleanup
+ fs.delete(dir1, true);
+ fs.delete(dir2, true);
+ }
+ }
+
+ @Test
+ public void testListStatusIteratorWithPathNotFound() throws Exception {
+ Path root = new Path("/test");
+ try {
+ ofs.listStatusIterator(root);
+ Assert.fail("Should have thrown OMException");
+ } catch (OMException omEx) {
+ Assert.assertEquals("Volume test is not found",
+ OMException.ResultCodes.VOLUME_NOT_FOUND, omEx.getResult());
+ }
+ }
+
+ /**
+ * Tests listStatusIterator operation on root directory with different
+ * numbers of numDir.
+ */
+ @Test
+ public void testListStatusIteratorOnPageSize() throws Exception {
+ int[] pageSize = {
+ 1, LISTING_PAGE_SIZE, LISTING_PAGE_SIZE + 1,
+ LISTING_PAGE_SIZE - 1, LISTING_PAGE_SIZE + LISTING_PAGE_SIZE / 2,
+ LISTING_PAGE_SIZE + LISTING_PAGE_SIZE
+ };
+ for (int numDir : pageSize) {
+ int range = numDir / LISTING_PAGE_SIZE;
+ switch (range) {
+ case 0:
+ listStatusIterator(numDir);
+ break;
+ case 1:
+ listStatusIterator(numDir);
+ break;
+ case 2:
+ listStatusIterator(numDir);
+ break;
+ default:
+ listStatusIterator(numDir);
+ }
+ }
+ }
+
+ private void listStatusIterator(int numDirs) throws IOException {
+ Path root = new Path("/" + volumeName + "/" + bucketName);
+ Set<String> paths = new TreeSet<>();
+ try {
+ for (int i = 0; i < numDirs; i++) {
+ Path p = new Path(root, String.valueOf(i));
+ fs.mkdirs(p);
+ paths.add(p.getName());
+ }
+
+ RemoteIterator<FileStatus> iterator = ofs.listStatusIterator(root);
+ int iCount = 0;
+ if (iterator != null) {
+ while (iterator.hasNext()) {
+ FileStatus fileStatus = iterator.next();
+ iCount++;
+ Assert.assertTrue(paths.contains(fileStatus.getPath().getName()));
+ }
+ }
+ Assert.assertEquals(
+ "Total directories listed do not match the existing directories",
+ numDirs, iCount);
+
+ } finally {
+ // Cleanup
+ for (int i = 0; i < numDirs; i++) {
+ Path p = new Path(root, String.valueOf(i));
+ fs.delete(p, true);
+ }
+ }
+ }
+
+ /**
+ * Tests listStatusIterator on a path with subdirs.
+ */
+ @Test
+ public void testListStatusIteratorOnSubDirs() throws Exception {
+ // Create the following key structure
+ // /dir1/dir11/dir111
+ // /dir1/dir12
+ // /dir1/dir12/file121
+ // /dir2
+ // listStatusIterator on /dir1 should return all its immediated subdirs only
+ // which are /dir1/dir11 and /dir1/dir12. Super child files/dirs
+ // (/dir1/dir12/file121 and /dir1/dir11/dir111) should not be returned by
+ // listStatusIterator.
+ Path dir1 = new Path(bucketPath, "dir1");
+ Path dir11 = new Path(dir1, "dir11");
+ Path dir111 = new Path(dir11, "dir111");
+ Path dir12 = new Path(dir1, "dir12");
+ Path file121 = new Path(dir12, "file121");
+ Path dir2 = new Path(bucketPath, "dir2");
+ try {
+ fs.mkdirs(dir111);
+ fs.mkdirs(dir12);
+ ContractTestUtils.touch(fs, file121);
+ fs.mkdirs(dir2);
+
+ RemoteIterator<FileStatus> it = ofs.listStatusIterator(dir1);
+ int iCount = 0;
+ while (it.hasNext()) {
+ iCount++;
+ FileStatus fileStatus = it.next();
+ Assert.assertNotNull(fileStatus);
+ Assert.assertNotEquals(fileStatus, dir12.toString());
+ // Verify that the two children of /dir1
+ // returned by listStatusIterator operation
+ // are /dir1/dir11 and /dir1/dir12.
+ Assert.assertTrue(
+ fileStatus.getPath().toUri().getPath().
+ equals(dir11.toString()) ||
+ fileStatus.getPath().toUri().getPath().
+ equals(dir12.toString()));
+ }
+ Assert.assertEquals(
+ "Iterator should return only the immediate children",
+ 2, iCount);
+ } finally {
+ // Cleanup
+ fs.delete(dir2, true);
+ fs.delete(dir1, true);
+ }
+ }
+
/**
* OFS: Helper function for tests. Return a volume name that doesn't exist.
*/
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
index 2e756a5e6a..04683ff760 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
@@ -864,6 +864,133 @@ public class BasicOzoneFileSystem extends FileSystem {
return super.listLocatedStatus(f);
}
+ @Override
+ public RemoteIterator<FileStatus> listStatusIterator(Path f)
+ throws IOException {
+ return new OzoneFileStatusIterator<>(f);
+ }
+
+ /**
+ * A private class implementation for iterating list of file status.
+ *
+ * @param <T> the type of the file status.
+ */
+ private final class OzoneFileStatusIterator<T extends FileStatus>
+ implements RemoteIterator<T> {
+ private List<FileStatus> thisListing;
+ private int i;
+ private Path p;
+ private T curStat = null;
+ private String startPath = "";
+
+ /**
+ * Constructor to initialize OzoneFileStatusIterator.
+ * Get the first batch of entry for iteration.
+ *
+ * @param p path to file/directory.
+ * @throws IOException
+ */
+ private OzoneFileStatusIterator(Path p) throws IOException {
+ this.p = p;
+ // fetch the first batch of entries in the directory
+ thisListing = listFileStatus(p, startPath);
+ if (thisListing != null && !thisListing.isEmpty()) {
+ startPath = pathToKey(
+ thisListing.get(thisListing.size() - 1).getPath());
+ LOG.debug("Got {} file status, next start path {}",
+ thisListing.size(), startPath);
+ }
+ i = 0;
+ }
+
+ /**
+ * @return true if next entry exists false otherwise.
+ * @throws IOException
+ */
+ @Override
+ public boolean hasNext() throws IOException {
+ while (curStat == null && hasNextNoFilter()) {
+ T next;
+ FileStatus fileStat = thisListing.get(i++);
+ next = (T) (fileStat);
+ curStat = next;
+ }
+ return curStat != null;
+ }
+
+ /**
+ * Checks the next available entry from partial listing if not exhausted
+ * or fetches new batch for listing.
+ *
+ * @return true if next entry exists false otherwise.
+ * @throws IOException
+ */
+ private boolean hasNextNoFilter() throws IOException {
+ if (thisListing == null) {
+ return false;
+ }
+ if (i >= thisListing.size()) {
+ if (startPath != null && (thisListing.size() == LISTING_PAGE_SIZE ||
+ thisListing.size() == LISTING_PAGE_SIZE - 1)) {
+ // current listing is exhausted & fetch a new listing
+ thisListing = listFileStatus(p, startPath);
+ if (thisListing != null && !thisListing.isEmpty()) {
+ startPath = pathToKey(
+ thisListing.get(thisListing.size() - 1).getPath());
+ LOG.debug("Got {} file status, next start path {}",
+ thisListing.size(), startPath);
+ } else {
+ return false;
+ }
+ i = 0;
+ }
+ }
+ return (i < thisListing.size());
+ }
+
+ /**
+ * @return next entry.
+ * @throws IOException
+ */
+ @Override
+ public T next() throws IOException {
+ if (hasNext()) {
+ T tmp = curStat;
+ curStat = null;
+ return tmp;
+ }
+ throw new java.util.NoSuchElementException("No more entry in " + p);
+ }
+ }
+
+ /**
+ * Get all the file status for input path and startPath.
+ *
+ * @param f
+ * @param startPath
+ * @return list of file status.
+ * @throws IOException
+ */
+ private List<FileStatus> listFileStatus(Path f, String startPath)
+ throws IOException {
+ incrementCounter(Statistic.INVOCATION_LIST_STATUS, 1);
+ statistics.incrementReadOps(1);
+ LOG.trace("listFileStatus() path:{}", f);
+ List<FileStatus> statusList;
+ statusList =
+ adapter.listStatus(pathToKey(f), false, startPath,
+ LISTING_PAGE_SIZE, uri, workingDir, getUsername())
+ .stream()
+ .map(this::convertFileStatus)
+ .collect(Collectors.toList());
+
+ if (!statusList.isEmpty() && !startPath.isEmpty()) {
+ // Excluding the 1st file status element from list.
+ statusList.remove(0);
+ }
+ return statusList;
+ }
+
/**
* Turn a path (relative or otherwise) into an Ozone key.
*
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
index fbd800f203..28491d66fd 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
@@ -966,6 +966,133 @@ public class BasicRootedOzoneFileSystem extends FileSystem {
return super.listLocatedStatus(f);
}
+ @Override
+ public RemoteIterator<FileStatus> listStatusIterator(Path f)
+ throws IOException {
+ return new OzoneFileStatusIterator<>(f);
+ }
+
+ /**
+ * A private class implementation for iterating list of file status.
+ *
+ * @param <T> the type of the file status.
+ */
+ private final class OzoneFileStatusIterator<T extends FileStatus>
+ implements RemoteIterator<T> {
+ private List<FileStatus> thisListing;
+ private int i;
+ private Path p;
+ private T curStat = null;
+ private String startPath = "";
+
+ /**
+ * Constructor to initialize OzoneFileStatusIterator.
+ * Get the first batch of entry for iteration.
+ *
+ * @param p path to file/directory.
+ * @throws IOException
+ */
+ private OzoneFileStatusIterator(Path p) throws IOException {
+ this.p = p;
+ // fetch the first batch of entries in the directory
+ thisListing = listFileStatus(p, startPath);
+ if (thisListing != null && !thisListing.isEmpty()) {
+ startPath = pathToKey(
+ thisListing.get(thisListing.size() - 1).getPath());
+ LOG.debug("Got {} file status, next start path {}",
+ thisListing.size(), startPath);
+ }
+ i = 0;
+ }
+
+ /**
+ * @return true if next entry exists false otherwise.
+ * @throws IOException
+ */
+ @Override
+ public boolean hasNext() throws IOException {
+ while (curStat == null && hasNextNoFilter()) {
+ T next;
+ FileStatus fileStat = thisListing.get(i++);
+ next = (T) (fileStat);
+ curStat = next;
+ }
+ return curStat != null;
+ }
+
+ /**
+ * Checks the next available entry from partial listing if not exhausted
+ * or fetches new batch for listing.
+ *
+ * @return true if next entry exists false otherwise.
+ * @throws IOException
+ */
+ private boolean hasNextNoFilter() throws IOException {
+ if (thisListing == null) {
+ return false;
+ }
+ if (i >= thisListing.size()) {
+ if (startPath != null && (thisListing.size() == LISTING_PAGE_SIZE ||
+ thisListing.size() == LISTING_PAGE_SIZE - 1)) {
+ // current listing is exhausted & fetch a new listing
+ thisListing = listFileStatus(p, startPath);
+ if (thisListing != null && !thisListing.isEmpty()) {
+ startPath = pathToKey(
+ thisListing.get(thisListing.size() - 1).getPath());
+ LOG.debug("Got {} file status, next start path {}",
+ thisListing.size(), startPath);
+ } else {
+ return false;
+ }
+ i = 0;
+ }
+ }
+ return (i < thisListing.size());
+ }
+
+ /**
+ * @return next entry.
+ * @throws IOException
+ */
+ @Override
+ public T next() throws IOException {
+ if (hasNext()) {
+ T tmp = curStat;
+ curStat = null;
+ return tmp;
+ }
+ throw new java.util.NoSuchElementException("No more entry in " + p);
+ }
+ }
+
+ /**
+ * Get all the file status for input path and startPath.
+ *
+ * @param f
+ * @param startPath
+ * @return list of file status.
+ * @throws IOException
+ */
+ private List<FileStatus> listFileStatus(Path f, String startPath)
+ throws IOException {
+ incrementCounter(Statistic.INVOCATION_LIST_STATUS, 1);
+ statistics.incrementReadOps(1);
+ LOG.trace("listFileStatus() path:{}", f);
+ List<FileStatus> statusList;
+ statusList =
+ adapter.listStatus(pathToKey(f), false, startPath,
+ LISTING_PAGE_SIZE, uri, workingDir, getUsername())
+ .stream()
+ .map(this::convertFileStatus)
+ .collect(Collectors.toList());
+
+ if (!statusList.isEmpty() && !startPath.isEmpty()) {
+ // Excluding the 1st file status element from list.
+ statusList.remove(0);
+ }
+ return statusList;
+ }
+
/**
* Turn a path (relative or otherwise) into an Ozone key.
*
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org