You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2021/04/06 16:23:46 UTC
[hadoop] branch trunk updated: HDFS-15909. Make fnmatch cross
platform (#2792)
This is an automated email from the ASF dual-hosted git repository.
inigoiri pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 42ddb5c HDFS-15909. Make fnmatch cross platform (#2792)
42ddb5c is described below
commit 42ddb5c6fedf78ec6c183b1b66380dc6ef06ab7c
Author: Gautham B A <ga...@gmail.com>
AuthorDate: Tue Apr 6 21:53:21 2021 +0530
HDFS-15909. Make fnmatch cross platform (#2792)
---
.../main/native/libhdfspp/lib/fs/CMakeLists.txt | 5 +--
.../src/main/native/libhdfspp/lib/fs/filesystem.cc | 12 +++----
.../main/native/libhdfspp/lib/x-platform/syscall.h | 11 +++++++
.../libhdfspp/lib/x-platform/syscall_linux.cc | 6 ++++
.../libhdfspp/lib/x-platform/syscall_windows.cc | 9 ++++++
.../libhdfspp/tests/x-platform/CMakeLists.txt | 7 ++++
.../x-platform/syscall_common_test.cc} | 37 ++++++++++++----------
.../x-platform/syscall_nix_test.cc} | 24 +++++++-------
.../x-platform/syscall_win_test.cc} | 24 +++++++-------
9 files changed, 85 insertions(+), 50 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/CMakeLists.txt
index 624cda5..d6ea248 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/CMakeLists.txt
@@ -16,6 +16,7 @@
# limitations under the License.
#
-add_library(fs_obj OBJECT filesystem.cc filesystem_sync.cc filehandle.cc bad_datanode_tracker.cc namenode_operations.cc)
+add_library(fs_obj OBJECT $<TARGET_OBJECTS:x_platform_obj> filesystem.cc filesystem_sync.cc filehandle.cc bad_datanode_tracker.cc namenode_operations.cc)
+target_include_directories(fs_obj PRIVATE ../lib)
add_dependencies(fs_obj proto)
-add_library(fs $<TARGET_OBJECTS:fs_obj>)
+add_library(fs $<TARGET_OBJECTS:fs_obj> $<TARGET_OBJECTS:x_platform_obj>)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
index ba75e86..741d6c7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
@@ -25,12 +25,12 @@
#include <limits>
#include <future>
#include <tuple>
-#include <iostream>
#include <pwd.h>
-#include <fnmatch.h>
#include <boost/asio/ip/tcp.hpp>
+#include "x-platform/syscall.h"
+
#define FMT_THIS_ADDR "this=" << (void*)this
namespace hdfs {
@@ -722,8 +722,8 @@ void FileSystemImpl::FindShim(const Status &stat, const std::vector<StatInfo> &
for (StatInfo const& si : stat_infos) {
//If we are at the last depth and it matches both path and name, we need to output it.
if (operational_state->depth == shared_state->dirs.size() - 2
- && !fnmatch(shared_state->dirs[operational_state->depth + 1].c_str(), si.path.c_str(), 0)
- && !fnmatch(shared_state->name.c_str(), si.path.c_str(), 0)) {
+ && XPlatform::Syscall::FnMatch(shared_state->dirs[operational_state->depth + 1], si.path)
+ && XPlatform::Syscall::FnMatch(shared_state->name, si.path)) {
outputs.push_back(si);
}
//Skip if not directory
@@ -731,7 +731,7 @@ void FileSystemImpl::FindShim(const Status &stat, const std::vector<StatInfo> &
continue;
}
//Checking for a match with the path at the current depth
- if(!fnmatch(shared_state->dirs[operational_state->depth + 1].c_str(), si.path.c_str(), 0)){
+ if(XPlatform::Syscall::FnMatch(shared_state->dirs[operational_state->depth + 1], si.path)) {
//Launch a new requests for every matched directory
shared_state->outstanding_requests++;
auto callback = [this, si, operational_state, shared_state](const Status &stat, const std::vector<StatInfo> & stat_infos, bool has_more) {
@@ -755,7 +755,7 @@ void FileSystemImpl::FindShim(const Status &stat, const std::vector<StatInfo> &
nn_.GetListing(si.full_path, callback);
}
//All names that match the specified name are saved to outputs
- if(!fnmatch(shared_state->name.c_str(), si.path.c_str(), 0)){
+ if(XPlatform::Syscall::FnMatch(shared_state->name, si.path)) {
outputs.push_back(si);
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h
index d162f6f..297aceb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h
@@ -48,6 +48,17 @@ class Syscall {
*/
static int WriteToStdout(const char* message);
+ /**
+ * Checks whether the {@link str} argument matches the {@link pattern}
+ * argument, which is a shell wildcard pattern.
+ *
+ * @param pattern The wildcard pattern to use.
+ * @param str The string to match.
+ * @returns A boolean indicating whether the given {@link str}
+ * matches {@link pattern}.
+ */
+ static bool FnMatch(const std::string& pattern, const std::string& str);
+
private:
static bool WriteToStdoutImpl(const char* message);
};
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
index e556d99..2c51dbf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
@@ -16,6 +16,7 @@
* limitations under the License.
*/
+#include <fnmatch.h>
#include <unistd.h>
#include <cstring>
@@ -30,6 +31,11 @@ int XPlatform::Syscall::WriteToStdout(const char* message) {
return WriteToStdoutImpl(message) ? 1 : 0;
}
+bool XPlatform::Syscall::FnMatch(const std::string& pattern,
+ const std::string& str) {
+ return fnmatch(pattern.c_str(), str.c_str(), 0) == 0;
+}
+
bool XPlatform::Syscall::WriteToStdoutImpl(const char* message) {
const auto message_len = strlen(message);
const auto result = write(1, message, message_len);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc
index 06b0031..dc9ba63 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc
@@ -16,10 +16,13 @@
* limitations under the License.
*/
+#include <Shlwapi.h>
#include <Windows.h>
#include "syscall.h"
+#pragma comment(lib, "Shlwapi.lib")
+
bool XPlatform::Syscall::WriteToStdout(const std::string& message) {
return WriteToStdoutImpl(message.c_str());
}
@@ -28,6 +31,12 @@ int XPlatform::Syscall::WriteToStdout(const char* message) {
return WriteToStdoutImpl(message) ? 1 : 0;
}
+bool XPlatform::Syscall::FnMatch(const std::string& pattern,
+ const std::string& str) {
+ return PathMatchSpecA(static_cast<LPCSTR>(str.c_str()),
+ static_cast<LPCSTR>(pattern.c_str())) == TRUE;
+}
+
bool XPlatform::Syscall::WriteToStdoutImpl(const char* message) {
auto* const stdout_handle = GetStdHandle(STD_OUTPUT_HANDLE);
if (stdout_handle == INVALID_HANDLE_VALUE || stdout_handle == nullptr) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt
index ac9f8fb..6a7d0be 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt
@@ -18,9 +18,16 @@
if(WIN32)
add_executable(x_platform_utils_test $<TARGET_OBJECTS:x_platform_obj> utils_common_test.cc utils_test_main.cc utils_win_test.cc)
+ add_executable(x_platform_syscall_test $<TARGET_OBJECTS:x_platform_obj> syscall_common_test.cc utils_test_main.cc syscall_win_test.cc)
else(WIN32)
add_executable(x_platform_utils_test $<TARGET_OBJECTS:x_platform_obj> utils_common_test.cc utils_test_main.cc utils_nix_test.cc)
+ add_executable(x_platform_syscall_test $<TARGET_OBJECTS:x_platform_obj> syscall_common_test.cc utils_test_main.cc syscall_nix_test.cc)
endif(WIN32)
+
target_include_directories(x_platform_utils_test PRIVATE ${LIBHDFSPP_LIB_DIR})
target_link_libraries(x_platform_utils_test gmock_main)
add_test(x_platform_utils_test x_platform_utils_test)
+
+target_include_directories(x_platform_syscall_test PRIVATE ${LIBHDFSPP_LIB_DIR})
+target_link_libraries(x_platform_syscall_test gmock_main)
+add_test(x_platform_syscall_test x_platform_syscall_test)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc
similarity index 50%
copy from hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc
copy to hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc
index 06b0031..04da29a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc
@@ -16,27 +16,32 @@
* limitations under the License.
*/
-#include <Windows.h>
+#include <gtest/gtest.h>
-#include "syscall.h"
+#include <string>
-bool XPlatform::Syscall::WriteToStdout(const std::string& message) {
- return WriteToStdoutImpl(message.c_str());
+#include "x-platform/syscall.h"
+
+TEST(XPlatformSyscall, FnMatchBasicAsterisk) {
+ const std::string pattern("a*.doc");
+ const std::string str("abcd.doc");
+ EXPECT_TRUE(XPlatform::Syscall::FnMatch(pattern, str));
}
-int XPlatform::Syscall::WriteToStdout(const char* message) {
- return WriteToStdoutImpl(message) ? 1 : 0;
+TEST(XPlatformSyscall, FnMatchBasicQuestionMark) {
+ const std::string pattern("a?.doc");
+ const std::string str("ab.doc");
+ EXPECT_TRUE(XPlatform::Syscall::FnMatch(pattern, str));
}
-bool XPlatform::Syscall::WriteToStdoutImpl(const char* message) {
- auto* const stdout_handle = GetStdHandle(STD_OUTPUT_HANDLE);
- if (stdout_handle == INVALID_HANDLE_VALUE || stdout_handle == nullptr) {
- return false;
- }
+TEST(XPlatformSyscall, FnMatchNegativeAsterisk) {
+ const std::string pattern("a*.doc");
+ const std::string str("bcd.doc");
+ EXPECT_FALSE(XPlatform::Syscall::FnMatch(pattern, str));
+}
- unsigned long bytes_written = 0;
- const auto message_len = lstrlen(message);
- const auto result =
- WriteFile(stdout_handle, message, message_len, &bytes_written, nullptr);
- return result && static_cast<unsigned long>(message_len) == bytes_written;
+TEST(XPlatformSyscall, FnMatchNegativeQuestionMark) {
+ const std::string pattern("a?.doc");
+ const std::string str("abc.doc");
+ EXPECT_FALSE(XPlatform::Syscall::FnMatch(pattern, str));
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_nix_test.cc
similarity index 61%
copy from hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
copy to hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_nix_test.cc
index e556d99..f2c753f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_nix_test.cc
@@ -16,22 +16,20 @@
* limitations under the License.
*/
-#include <unistd.h>
+#include <gtest/gtest.h>
-#include <cstring>
+#include <string>
-#include "syscall.h"
+#include "x-platform/syscall.h"
-bool XPlatform::Syscall::WriteToStdout(const std::string& message) {
- return WriteToStdoutImpl(message.c_str());
+TEST(XPlatformSyscall, FnMatchBasicPath) {
+ const std::string pattern("*.doc");
+ const std::string str("some/path/abcd.doc");
+ EXPECT_TRUE(XPlatform::Syscall::FnMatch(pattern, str));
}
-int XPlatform::Syscall::WriteToStdout(const char* message) {
- return WriteToStdoutImpl(message) ? 1 : 0;
-}
-
-bool XPlatform::Syscall::WriteToStdoutImpl(const char* message) {
- const auto message_len = strlen(message);
- const auto result = write(1, message, message_len);
- return result == static_cast<ssize_t>(message_len);
+TEST(XPlatformSyscall, FnMatchNegativePath) {
+ const std::string pattern("x*.doc");
+ const std::string str("y/abcd.doc");
+ EXPECT_FALSE(XPlatform::Syscall::FnMatch(pattern, str));
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_win_test.cc
similarity index 61%
copy from hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
copy to hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_win_test.cc
index e556d99..2d7393f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_win_test.cc
@@ -16,22 +16,20 @@
* limitations under the License.
*/
-#include <unistd.h>
+#include <gtest/gtest.h>
-#include <cstring>
+#include <string>
-#include "syscall.h"
+#include "x-platform/syscall.h"
-bool XPlatform::Syscall::WriteToStdout(const std::string& message) {
- return WriteToStdoutImpl(message.c_str());
+TEST(XPlatformSyscall, FnMatchBasicPath) {
+ const std::string pattern("*.doc");
+ const std::string str(R"(some\path\abcd.doc)");
+ EXPECT_TRUE(XPlatform::Syscall::FnMatch(pattern, str));
}
-int XPlatform::Syscall::WriteToStdout(const char* message) {
- return WriteToStdoutImpl(message) ? 1 : 0;
-}
-
-bool XPlatform::Syscall::WriteToStdoutImpl(const char* message) {
- const auto message_len = strlen(message);
- const auto result = write(1, message, message_len);
- return result == static_cast<ssize_t>(message_len);
+TEST(XPlatformSyscall, FnMatchNegativePath) {
+ const std::string pattern("x*.doc");
+ const std::string str(R"(y\abcd.doc)");
+ EXPECT_FALSE(XPlatform::Syscall::FnMatch(pattern, str));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org