You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/12/06 00:24:19 UTC
(doris) branch master updated: [fix](hdfs) Fix hdfsExists that return staled root cause (#27991)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 73404405e68 [fix](hdfs) Fix hdfsExists that return staled root cause (#27991)
73404405e68 is described below
commit 73404405e68f92b9363a03f78854c78f985f01d4
Author: walter <w4...@gmail.com>
AuthorDate: Wed Dec 6 08:24:13 2023 +0800
[fix](hdfs) Fix hdfsExists that return staled root cause (#27991)
The HDFS native client won't clear the last exception as expected so `hdfsGetLastExceptionRootCause` might return a staled root cause. This PR saves the last root cause here and verifies after hdfsExists returns a non-zero code.
---
be/src/io/fs/hdfs_file_system.cpp | 19 ++++++++++++++++---
1 file changed, 16 insertions(+), 3 deletions(-)
diff --git a/be/src/io/fs/hdfs_file_system.cpp b/be/src/io/fs/hdfs_file_system.cpp
index 1e71ade9344..7e0062d2430 100644
--- a/be/src/io/fs/hdfs_file_system.cpp
+++ b/be/src/io/fs/hdfs_file_system.cpp
@@ -227,15 +227,28 @@ Status HdfsFileSystem::delete_internal(const Path& path, int is_recursive) {
Status HdfsFileSystem::exists_impl(const Path& path, bool* res) const {
CHECK_HDFS_HANDLE(_fs_handle);
Path real_path = convert_path(path, _fs_name);
+#ifdef USE_HADOOP_HDFS
+ // HACK: the HDFS native client won't clear the last exception as expected so
+ // `hdfsGetLastExceptionRootCause` might return a staled root cause. Save the
+ // last root cause here and verify after hdfsExists returns a non-zero code.
+ //
+ // See details:
+ // https://github.com/apache/hadoop/blob/5cda162a804fb0cfc2a5ac0058ab407662c5fb00/
+ // hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c#L795
+ char* former_root_cause = hdfsGetLastExceptionRootCause();
+#endif
int is_exists = hdfsExists(_fs_handle->hdfs_fs, real_path.string().c_str());
#ifdef USE_HADOOP_HDFS
// when calling hdfsExists() and return non-zero code,
// if root_cause is nullptr, which means the file does not exist.
// if root_cause is not nullptr, which means it encounter other error, should return.
// NOTE: not for libhdfs3 since it only runs on MaxOS, don't have to support it.
- char* root_cause = hdfsGetLastExceptionRootCause();
- if (root_cause != nullptr) {
- return Status::IOError("failed to check path existence {}: {}", path.native(), root_cause);
+ if (is_exists != 0) {
+ char* root_cause = hdfsGetLastExceptionRootCause();
+ if (root_cause != nullptr && root_cause != former_root_cause) {
+ return Status::IOError("failed to check path existence {}: {}", path.native(),
+ root_cause);
+ }
}
#endif
*res = (is_exists == 0);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org