[hawq] branch master updated: HAWQ-1769. Fix DirectoryIterator of libhdfs3

[hu...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

huor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hawq.git


The following commit(s) were added to refs/heads/master by this push:
     new b3f60b3  HAWQ-1769. Fix DirectoryIterator of libhdfs3
b3f60b3 is described below

commit b3f60b333fbbdc43429fbb35dc33a35c4f415a52
Author: qinzuoyan <qi...@xiaomi.com>
AuthorDate: Tue Oct 20 23:25:55 2020 +0800

    HAWQ-1769. Fix DirectoryIterator of libhdfs3
---
 depends/libhdfs3/src/client/DirectoryIterator.cpp | 32 +++++++++++------------
 depends/libhdfs3/src/client/DirectoryIterator.h   |  1 +
 depends/libhdfs3/src/server/Namenode.h            |  2 ++
 3 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/depends/libhdfs3/src/client/DirectoryIterator.cpp b/depends/libhdfs3/src/client/DirectoryIterator.cpp
index 40b39a4..7cfb97f 100644
--- a/depends/libhdfs3/src/client/DirectoryIterator.cpp
+++ b/depends/libhdfs3/src/client/DirectoryIterator.cpp
@@ -28,17 +28,17 @@
 namespace Hdfs {
 
 DirectoryIterator::DirectoryIterator() :
-    needLocations(false), filesystem(NULL), next(0) {
+    needLocations(false), hasMore(true), filesystem(NULL), next(0) {
 }
 
 DirectoryIterator::DirectoryIterator(Hdfs::Internal::FileSystemImpl * const fs,
                                      std::string path, bool needLocations) :
-    needLocations(needLocations), filesystem(fs), next(0), path(path) {
+    needLocations(needLocations), hasMore(true), filesystem(fs), next(0), path(path) {
 }
 
 DirectoryIterator::DirectoryIterator(const DirectoryIterator & it) :
-    needLocations(it.needLocations), filesystem(it.filesystem), next(it.next), path(it.path), startAfter(
-        it.startAfter), lists(it.lists) {
+    needLocations(it.needLocations), hasMore(it.hasMore), filesystem(it.filesystem),
+    next(it.next), path(it.path), startAfter(it.startAfter), lists(it.lists) {
 }
 
 DirectoryIterator & DirectoryIterator::operator =(const DirectoryIterator & it) {
@@ -47,6 +47,7 @@ DirectoryIterator & DirectoryIterator::operator =(const DirectoryIterator & it)
     }
 
     needLocations = it.needLocations;
+    hasMore = it.hasMore;
     filesystem = it.filesystem;
     next = it.next;
     path = it.path;
@@ -56,36 +57,33 @@ DirectoryIterator & DirectoryIterator::operator =(const DirectoryIterator & it)
 }
 
 bool DirectoryIterator::getListing() {
-    bool more;
-
-    if (NULL == filesystem) {
+    if (NULL == filesystem || !hasMore) {
         return false;
     }
 
     next = 0;
     lists.clear();
-    more = filesystem->getListing(path, startAfter, needLocations, lists);
 
-    if (!lists.empty()) {
+    hasMore = filesystem->getListing(path, startAfter, needLocations, lists);
+
+    if (hasMore && !lists.empty()) {
         startAfter = lists.back().getPath();
     }
 
-    return more || !lists.empty();
+    return !lists.empty();
 }
 
 bool DirectoryIterator::hasNext() {
-    if (next >= lists.size()) {
-        return getListing();
+    if (next < lists.size()) {
+        return true;
     }
 
-    return true;
+    return getListing();
 }
 
 Hdfs::FileStatus DirectoryIterator::getNext() {
-    if (next >= lists.size()) {
-        if (!getListing()) {
-            THROW(HdfsIOException, "End of the dir flow");
-        }
+    if (!hasNext()) {
+        THROW(HdfsIOException, "End of the dir flow");
     }
 
     return lists[next++];
diff --git a/depends/libhdfs3/src/client/DirectoryIterator.h b/depends/libhdfs3/src/client/DirectoryIterator.h
index 932f086..513241f 100644
--- a/depends/libhdfs3/src/client/DirectoryIterator.h
+++ b/depends/libhdfs3/src/client/DirectoryIterator.h
@@ -45,6 +45,7 @@ private:
 
 private:
     bool needLocations;
+    bool hasMore;
     Hdfs::Internal::FileSystemImpl * filesystem;
     size_t next;
     std::string path;
diff --git a/depends/libhdfs3/src/server/Namenode.h b/depends/libhdfs3/src/server/Namenode.h
index ad213e8..0628788 100644
--- a/depends/libhdfs3/src/server/Namenode.h
+++ b/depends/libhdfs3/src/server/Namenode.h
@@ -514,6 +514,8 @@ public:
      * @param a partial listing starting after startAfter
      * @param dl append the returned directories.
      *
+     * @return true if there are more remaining entries.
+     *
      * @throw AccessControlException permission denied
      * @throw FileNotFoundException file <code>src</code> is not found
      * @throw UnresolvedLinkException If <code>src</code> contains a symlink