(doris) branch branch-2.1 updated: [fix](cooldown) Fix hdfs path (#33315)

[pl...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

plat1ko pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-2.1 by this push:
     new 97850cf2bbf [fix](cooldown) Fix hdfs path (#33315)
97850cf2bbf is described below

commit 97850cf2bbf6d3fac993094e70731af5966f94f9
Author: plat1ko <pl...@gmail.com>
AuthorDate: Tue Apr 9 12:55:53 2024 +0800

    [fix](cooldown) Fix hdfs path (#33315)
---
 be/src/io/fs/hdfs_file_system.cpp |  7 ++++---
 be/src/util/hdfs_util.cpp         | 22 ++++++++++++----------
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/be/src/io/fs/hdfs_file_system.cpp b/be/src/io/fs/hdfs_file_system.cpp
index 5b5dbba16b2..22b4784ea61 100644
--- a/be/src/io/fs/hdfs_file_system.cpp
+++ b/be/src/io/fs/hdfs_file_system.cpp
@@ -283,11 +283,12 @@ Status HdfsFileSystem::list_impl(const Path& path, bool only_file, std::vector<F
         if (only_file && file.mKind == kObjectKindDirectory) {
             continue;
         }
-        FileInfo file_info;
-        file_info.file_name = file.mName;
+        auto& file_info = files->emplace_back();
+        std::string_view fname(file.mName);
+        fname.remove_prefix(fname.rfind('/') + 1);
+        file_info.file_name = fname;
         file_info.file_size = file.mSize;
         file_info.is_file = (file.mKind != kObjectKindDirectory);
-        files->emplace_back(std::move(file_info));
     }
     hdfsFreeFileInfo(hdfs_file_info, numEntries);
     return Status::OK();
diff --git a/be/src/util/hdfs_util.cpp b/be/src/util/hdfs_util.cpp
index 6e99fdea3d3..794c53f15e4 100644
--- a/be/src/util/hdfs_util.cpp
+++ b/be/src/util/hdfs_util.cpp
@@ -42,17 +42,19 @@ hdfsFS HDFSHandle::create_hdfs_fs(HDFSCommonBuilder& hdfs_builder) {
 }
 
 Path convert_path(const Path& path, const std::string& namenode) {
-    Path real_path(path);
-    if (path.string().find(namenode) != std::string::npos) {
-        std::string real_path_str = path.string().substr(namenode.size());
-        if (!real_path_str.starts_with("/")) {
-            // The real path must starts with "/"
-            // Or the hadoop client will add a prefix like "/user/hadoop".
-            real_path_str = "/" + real_path_str;
-        }
-        real_path = real_path_str;
+    std::string fs_path;
+    if (path.native().starts_with(namenode)) {
+        // `path` is URI format, remove the namenode part in `path`
+        fs_path = path.native().substr(namenode.size());
+    } else {
+        fs_path = path;
     }
-    return real_path;
+
+    // Always use absolute path (start with '/') in hdfs
+    if (fs_path.empty() || fs_path[0] != '/') {
+        fs_path.insert(fs_path.begin(), '/');
+    }
+    return fs_path;
 }
 
 bool is_hdfs(const std::string& path_or_fs) {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org