You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2018/03/22 21:21:04 UTC
[01/52] [abbrv] hadoop git commit: HDFS-10785: libhdfs++: Implement
the rest of the tools. Contributed by Anatoli Schein [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/HDFS-8707 24413ad96 -> 8620d2bdf (forced update)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc
new file mode 100644
index 0000000..133bbc9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc
@@ -0,0 +1,115 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include "tools_common.h"
+
+namespace hdfs {
+
+ std::shared_ptr<hdfs::FileSystem> doConnect(hdfs::URI & uri, bool max_timeout) {
+ hdfs::Options options;
+ //Setting the config path to the default: "$HADOOP_CONF_DIR" or "/etc/hadoop/conf"
+ hdfs::ConfigurationLoader loader;
+ //Loading default config files core-site.xml and hdfs-site.xml from the config path
+ hdfs::optional<HdfsConfiguration> config = loader.LoadDefaultResources<HdfsConfiguration>();
+ //TODO: HDFS-9539 - after this is resolved, valid config will always be returned.
+ if(config){
+ //Loading options from the config
+ options = config->GetOptions();
+ }
+ if(max_timeout){
+ //TODO: HDFS-9539 - until then we increase the time-out to allow all recursive async calls to finish
+ options.rpc_timeout = std::numeric_limits<int>::max();
+ }
+ IoService * io_service = IoService::New();
+ //Wrapping fs into a shared pointer to guarantee deletion
+ std::shared_ptr<hdfs::FileSystem> fs(hdfs::FileSystem::New(io_service, "", options));
+ if (!fs) {
+ std::cerr << "Could not create FileSystem object. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ Status status;
+ //Check if the user supplied the host
+ if(!uri.get_host().empty()){
+ //If port is supplied we use it, otherwise we use the empty string so that it will be looked up in configs.
+ std::string port = (uri.get_port()) ? std::to_string(uri.get_port().value()) : "";
+ status = fs->Connect(uri.get_host(), port);
+ if (!status.ok()) {
+ std::cerr << "Could not connect to " << uri.get_host() << ":" << port << ". " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ } else {
+ status = fs->ConnectToDefaultFs();
+ if (!status.ok()) {
+ if(!options.defaultFS.get_host().empty()){
+ std::cerr << "Error connecting to " << options.defaultFS << ". " << status.ToString() << std::endl;
+ } else {
+ std::cerr << "Error connecting to the cluster: defaultFS is empty. " << status.ToString() << std::endl;
+ }
+ exit(EXIT_FAILURE);
+ }
+ }
+ return fs;
+ }
+
+ #define BUF_SIZE 1048576 //1 MB
+ static char input_buffer[BUF_SIZE];
+
+ void readFile(std::shared_ptr<hdfs::FileSystem> fs, std::string path, off_t offset, std::FILE* dst_file, bool to_delete) {
+ ssize_t total_bytes_read = 0;
+ size_t last_bytes_read = 0;
+
+ hdfs::FileHandle *file_raw = nullptr;
+ hdfs::Status status = fs->Open(path, &file_raw);
+ if (!status.ok()) {
+ std::cerr << "Could not open file " << path << ". " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ //wrapping file_raw into a unique pointer to guarantee deletion
+ std::unique_ptr<hdfs::FileHandle> file(file_raw);
+
+ do{
+ //Reading file chunks
+ status = file->PositionRead(input_buffer, sizeof(input_buffer), offset, &last_bytes_read);
+ if(status.ok()) {
+ //Writing file chunks to stdout
+ fwrite(input_buffer, last_bytes_read, 1, dst_file);
+ total_bytes_read += last_bytes_read;
+ offset += last_bytes_read;
+ } else {
+ if(status.is_invalid_offset()){
+ //Reached the end of the file
+ if(to_delete) {
+ //Deleting the file (recursive set to false)
+ hdfs::Status status = fs->Delete(path, false);
+ if (!status.ok()) {
+ std::cerr << "Error deleting the source file: " << path
+ << " " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ }
+ break;
+ } else {
+ std::cerr << "Error reading the file: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ }
+ } while (last_bytes_read > 0);
+ return;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cpp
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cpp b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cpp
deleted file mode 100644
index af882ce..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cpp
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-*/
-
-#include "tools_common.h"
-
-namespace hdfs {
-
- std::shared_ptr<hdfs::Options> getOptions() {
- std::shared_ptr<hdfs::Options> options = std::make_shared<hdfs::Options>();
- //Setting the config path to the default: "$HADOOP_CONF_DIR" or "/etc/hadoop/conf"
- hdfs::ConfigurationLoader loader;
- //Loading default config files core-site.xml and hdfs-site.xml from the config path
- hdfs::optional<HdfsConfiguration> config = loader.LoadDefaultResources<HdfsConfiguration>();
- //TODO: HDFS-9539 - after this is resolved, valid config will always be returned.
- if(config){
- //Loading options from the config
- *options = config->GetOptions();
- }
- return options;
- }
-
- std::shared_ptr<hdfs::FileSystem> doConnect(hdfs::URI & uri, hdfs::Options & options) {
- IoService * io_service = IoService::New();
- //Wrapping fs into a shared pointer to guarantee deletion
- std::shared_ptr<hdfs::FileSystem> fs(hdfs::FileSystem::New(io_service, "", options));
- if (!fs) {
- std::cerr << "Could not create FileSystem object. " << std::endl;
- exit(EXIT_FAILURE);
- }
- Status status;
- //Check if the user supplied the host
- if(!uri.get_host().empty()){
- //If port is supplied we use it, otherwise we use the empty string so that it will be looked up in configs.
- std::string port = (uri.get_port()) ? std::to_string(uri.get_port().value()) : "";
- status = fs->Connect(uri.get_host(), port);
- if (!status.ok()) {
- std::cerr << "Could not connect to " << uri.get_host() << ":" << port << ". " << status.ToString() << std::endl;
- exit(EXIT_FAILURE);
- }
- } else {
- status = fs->ConnectToDefaultFs();
- if (!status.ok()) {
- if(!options.defaultFS.get_host().empty()){
- std::cerr << "Error connecting to " << options.defaultFS << ". " << status.ToString() << std::endl;
- } else {
- std::cerr << "Error connecting to the cluster: defaultFS is empty. " << status.ToString() << std::endl;
- }
- exit(EXIT_FAILURE);
- }
- }
- return fs;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.h
index 858fc4b..f7c5e09 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.h
@@ -23,17 +23,15 @@
#include "hdfspp/hdfspp.h"
#include "common/hdfs_configuration.h"
#include "common/configuration_loader.h"
-
#include <mutex>
namespace hdfs {
- //Pull configurations and get the Options object
- std::shared_ptr<hdfs::Options> getOptions();
-
//Build all necessary objects and perform the connection
- std::shared_ptr<hdfs::FileSystem> doConnect(hdfs::URI & uri, hdfs::Options & options);
+ std::shared_ptr<hdfs::FileSystem> doConnect(hdfs::URI & uri, bool max_timeout);
+ //Open HDFS file at offset, read it to destination file, optionally delete source file
+ void readFile(std::shared_ptr<hdfs::FileSystem> fs, std::string path, off_t offset, std::FILE* dst_file, bool to_delete);
}
#endif
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[52/52] [abbrv] hadoop git commit: HDFS-10787: libhdfs++: Public API
should expose configuration parser. Original patch contributed by Mitchell
Tracy, followup work and compile warning fixes contributed by Anatoli Shein.
Posted by jh...@apache.org.
HDFS-10787: libhdfs++: Public API should expose configuration parser. Original patch contributed by Mitchell Tracy, followup work and compile warning fixes contributed by Anatoli Shein.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/12942f67
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/12942f67
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/12942f67
Branch: refs/heads/HDFS-8707
Commit: 12942f679ae86720c79df033506c53dc1145aa91
Parents: eeb49d0
Author: James Clampffer <ja...@hp.com>
Authored: Thu Sep 7 10:50:23 2017 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../src/main/native/libhdfspp/CMakeLists.txt | 23 +-
.../native/libhdfspp/examples/cc/CMakeLists.txt | 2 +
.../libhdfspp/examples/cc/cat/CMakeLists.txt | 2 +-
.../native/libhdfspp/examples/cc/cat/cat.cc | 34 +--
.../libhdfspp/examples/cc/find/CMakeLists.txt | 2 +-
.../native/libhdfspp/examples/cc/find/find.cc | 34 +--
.../examples/cc/gendirs/CMakeLists.txt | 2 +-
.../libhdfspp/examples/cc/gendirs/gendirs.cc | 34 +--
.../libhdfspp/include/hdfspp/config_parser.h | 68 ++++++
.../native/libhdfspp/include/hdfspp/hdfspp.h | 1 +
.../native/libhdfspp/lib/bindings/c/hdfs.cc | 4 +-
.../native/libhdfspp/lib/common/CMakeLists.txt | 2 +-
.../libhdfspp/lib/common/config_parser.cc | 219 +++++++++++++++++++
.../libhdfspp/lib/common/configuration.cc | 4 +-
.../lib/common/configuration_loader.cc | 64 +++++-
.../libhdfspp/lib/common/configuration_loader.h | 10 +
.../lib/common/configuration_loader_impl.h | 5 +
.../lib/common/continuation/protobuf.h | 1 +
.../native/libhdfspp/lib/common/retry_policy.cc | 1 +
.../main/native/libhdfspp/lib/fs/filehandle.cc | 8 +-
.../main/native/libhdfspp/lib/fs/filesystem.h | 4 +-
.../libhdfspp/lib/fs/namenode_operations.h | 2 +-
.../native/libhdfspp/lib/reader/block_reader.cc | 1 +
.../libhdfspp/lib/rpc/rpc_connection_impl.cc | 2 -
.../libhdfspp/lib/rpc/rpc_connection_impl.h | 4 +-
.../main/native/libhdfspp/lib/rpc/rpc_engine.cc | 10 +-
.../native/libhdfspp/lib/rpc/sasl_engine.cc | 4 +-
.../native/libhdfspp/tests/configuration_test.h | 33 +++
.../libhdfspp/tests/hdfs_configuration_test.cc | 48 ++++
.../main/native/libhdfspp/tools/tools_common.cc | 30 ++-
.../main/native/libhdfspp/tools/tools_common.h | 3 -
31 files changed, 528 insertions(+), 133 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
index 2bcb49a..e9d87c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
@@ -30,6 +30,8 @@ cmake_minimum_required(VERSION 2.8)
enable_testing()
include (CTest)
+
+SET(BUILD_SHARED_HDFSPP TRUE CACHE STRING "BUILD_SHARED_HDFSPP defaulting to 'TRUE'")
SET(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH})
# If there's a better way to inform FindCyrusSASL.cmake, let's make this cleaner:
@@ -246,6 +248,7 @@ if(NEED_LINK_DL)
set(LIB_DL dl)
endif()
+set(LIBHDFSPP_VERSION "0.1.0")
set(LIBHDFSPP_ALL_OBJECTS $<TARGET_OBJECTS:bindings_c_obj> $<TARGET_OBJECTS:fs_obj> $<TARGET_OBJECTS:rpc_obj> $<TARGET_OBJECTS:reader_obj> $<TARGET_OBJECTS:proto_obj> $<TARGET_OBJECTS:connection_obj> $<TARGET_OBJECTS:common_obj> $<TARGET_OBJECTS:uriparser2_obj>)
if (HADOOP_BUILD)
hadoop_add_dual_library(hdfspp ${EMPTY_FILE_CC} ${LIBHDFSPP_ALL_OBJECTS})
@@ -256,6 +259,7 @@ if (HADOOP_BUILD)
${SASL_LIBRARIES}
${CMAKE_THREAD_LIBS_INIT}
)
+ set_target_properties(hdfspp PROPERTIES SOVERSION ${LIBHDFSPP_VERSION})
else (HADOOP_BUILD)
add_library(hdfspp_static STATIC ${EMPTY_FILE_CC} ${LIBHDFSPP_ALL_OBJECTS})
target_link_libraries(hdfspp_static
@@ -265,18 +269,11 @@ else (HADOOP_BUILD)
${SASL_LIBRARIES}
${CMAKE_THREAD_LIBS_INIT}
)
- add_library(hdfspp SHARED ${EMPTY_FILE_CC} ${LIBHDFSPP_ALL_OBJECTS})
- target_link_libraries(hdfspp_static
- ${LIB_DL}
- ${PROTOBUF_LIBRARY}
- ${OPENSSL_LIBRARIES}
- ${SASL_LIBRARIES}
- ${CMAKE_THREAD_LIBS_INIT}
- )
+ if(BUILD_SHARED_HDFSPP)
+ add_library(hdfspp SHARED ${EMPTY_FILE_CC} ${LIBHDFSPP_ALL_OBJECTS})
+ set_target_properties(hdfspp PROPERTIES SOVERSION ${LIBHDFSPP_VERSION})
+ endif(BUILD_SHARED_HDFSPP)
endif (HADOOP_BUILD)
-set(LIBHDFSPP_VERSION "0.1.0")
-set_target_properties(hdfspp PROPERTIES
- SOVERSION ${LIBHDFSPP_VERSION})
# Set up make install targets
# Can be installed to a particular location via "make DESTDIR=... install"
@@ -286,7 +283,9 @@ install(FILES ${LIBHDFSPP_HEADER_FILES} DESTINATION include/hdfspp)
install(FILES ${LIBHDFS_HEADER_FILES} DESTINATION include/hdfs)
install(TARGETS hdfspp_static ARCHIVE DESTINATION lib)
-install(TARGETS hdfspp LIBRARY DESTINATION lib)
+if(BUILD_SHARED_HDFSPP)
+ install(TARGETS hdfspp LIBRARY DESTINATION lib)
+endif(BUILD_SHARED_HDFSPP)
add_custom_target(
InstallToBuildDirectory
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/CMakeLists.txt
index 8893f03..1849779 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/CMakeLists.txt
@@ -16,6 +16,8 @@
# limitations under the License.
#
+include_directories( ../../tools )
+
add_subdirectory(cat)
add_subdirectory(gendirs)
add_subdirectory(find)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/CMakeLists.txt
index 479ff86..a5bcf76 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/CMakeLists.txt
@@ -24,4 +24,4 @@ include_directories( ${LIBHDFSPP_DIR}/include )
link_directories( ${LIBHDFSPP_DIR}/lib )
add_executable(cat cat.cc)
-target_link_libraries(cat hdfspp_static)
\ No newline at end of file
+target_link_libraries(cat tools_common hdfspp_static)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/cat.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/cat.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/cat.cc
index bca2a94..9d400e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/cat.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/cat.cc
@@ -31,10 +31,8 @@
**/
#include "hdfspp/hdfspp.h"
-#include "common/hdfs_configuration.h"
-#include "common/configuration_loader.h"
-
#include <google/protobuf/stubs/common.h>
+#include "tools_common.h"
const std::size_t BUF_SIZE = 1048576; //1 MB
static char input_buffer[BUF_SIZE];
@@ -46,35 +44,17 @@ int main(int argc, char *argv[]) {
}
std::string path = argv[1];
- hdfs::Options options;
- //Setting the config path to the default: "$HADOOP_CONF_DIR" or "/etc/hadoop/conf"
- hdfs::ConfigurationLoader loader;
- //Loading default config files core-site.xml and hdfs-site.xml from the config path
- hdfs::optional<hdfs::HdfsConfiguration> config = loader.LoadDefaultResources<hdfs::HdfsConfiguration>();
- //TODO: HDFS-9539 - after this is resolved, valid config will always be returned.
- if(config){
- //Loading options from the config
- options = config->GetOptions();
- }
- hdfs::IoService * io_service = hdfs::IoService::New();
- //Wrapping fs into a shared pointer to guarantee deletion
- std::shared_ptr<hdfs::FileSystem> fs(hdfs::FileSystem::New(io_service, "", options));
+ //Building a URI object from the given uri path
+ hdfs::URI uri = hdfs::parse_path_or_exit(path);
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
if (!fs) {
- std::cerr << "Could not connect the file system." << std::endl;
- exit(EXIT_FAILURE);
- }
- hdfs::Status status = fs->ConnectToDefaultFs();
- if (!status.ok()) {
- if(!options.defaultFS.get_host().empty()){
- std::cerr << "Error connecting to " << options.defaultFS << ". " << status.ToString() << std::endl;
- } else {
- std::cerr << "Error connecting to the cluster: defaultFS is empty. " << status.ToString() << std::endl;
- }
+ std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
}
hdfs::FileHandle *file_raw = nullptr;
- status = fs->Open(path, &file_raw);
+ hdfs::Status status = fs->Open(path, &file_raw);
if (!status.ok()) {
std::cerr << "Could not open file " << path << ". " << status.ToString() << std::endl;
exit(EXIT_FAILURE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/find/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/find/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/find/CMakeLists.txt
index 8210bb5..c833676 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/find/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/find/CMakeLists.txt
@@ -24,4 +24,4 @@ include_directories( ${LIBHDFSPP_DIR}/include )
link_directories( ${LIBHDFSPP_DIR}/lib )
add_executable(find find.cc)
-target_link_libraries(find hdfspp_static)
\ No newline at end of file
+target_link_libraries(find tools_common hdfspp_static)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/find/find.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/find/find.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/find/find.cc
index 21a731b..5373890 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/find/find.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/find/find.cc
@@ -38,11 +38,9 @@
**/
#include "hdfspp/hdfspp.h"
-#include "common/hdfs_configuration.h"
-#include "common/configuration_loader.h"
-
#include <google/protobuf/stubs/common.h>
#include <future>
+#include "tools_common.h"
void SyncFind(std::shared_ptr<hdfs::FileSystem> fs, const std::string &path, const std::string &name){
std::vector<hdfs::StatInfo> results;
@@ -119,32 +117,12 @@ int main(int argc, char *argv[]) {
std::string name = argv[2];
bool use_async = (std::stoi(argv[3]) != 0);
- hdfs::Options options;
- //Setting the config path to the default: "$HADOOP_CONF_DIR" or "/etc/hadoop/conf"
- hdfs::ConfigurationLoader loader;
- //Loading default config files core-site.xml and hdfs-site.xml from the config path
- hdfs::optional<hdfs::HdfsConfiguration> config = loader.LoadDefaultResources<hdfs::HdfsConfiguration>();
- //TODO: HDFS-9539 - after this is resolved, valid config will always be returned.
- if(config){
- //Loading options from the config
- options = config->GetOptions();
- }
- //TODO: HDFS-9539 - until then we increase the time-out to allow all recursive async calls to finish
- options.rpc_timeout = std::numeric_limits<int>::max();
- hdfs::IoService * io_service = hdfs::IoService::New();
- //Wrapping fs into a unique pointer to guarantee deletion
- std::shared_ptr<hdfs::FileSystem> fs(hdfs::FileSystem::New(io_service, "", options));
+ //Building a URI object from the given uri path
+ hdfs::URI uri = hdfs::parse_path_or_exit(path);
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, true);
if (!fs) {
- std::cerr << "Could not connect the file system." << std::endl;
- exit(EXIT_FAILURE);
- }
- hdfs::Status status = fs->ConnectToDefaultFs();
- if (!status.ok()) {
- if(!options.defaultFS.get_host().empty()){
- std::cerr << "Error connecting to " << options.defaultFS << ". " << status.ToString() << std::endl;
- } else {
- std::cerr << "Error connecting to the cluster: defaultFS is empty. " << status.ToString() << std::endl;
- }
+ std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/gendirs/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/gendirs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/gendirs/CMakeLists.txt
index 7fdc1b2..9bd8631 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/gendirs/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/gendirs/CMakeLists.txt
@@ -24,4 +24,4 @@ include_directories( ${LIBHDFSPP_DIR}/include )
link_directories( ${LIBHDFSPP_DIR}/lib )
add_executable(gendirs gendirs.cc)
-target_link_libraries(gendirs hdfspp_static)
\ No newline at end of file
+target_link_libraries(gendirs tools_common hdfspp_static)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/gendirs/gendirs.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/gendirs/gendirs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/gendirs/gendirs.cc
index c90abbd..35f7be9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/gendirs/gendirs.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/gendirs/gendirs.cc
@@ -37,11 +37,9 @@
**/
#include "hdfspp/hdfspp.h"
-#include "common/hdfs_configuration.h"
-#include "common/configuration_loader.h"
-
#include <google/protobuf/stubs/common.h>
#include <future>
+#include "tools_common.h"
#define DEFAULT_PERMISSIONS 0755
@@ -80,32 +78,12 @@ int main(int argc, char *argv[]) {
int depth = std::stoi(argv[2]);
int fanout = std::stoi(argv[3]);
- hdfs::Options options;
- //Setting the config path to the default: "$HADOOP_CONF_DIR" or "/etc/hadoop/conf"
- hdfs::ConfigurationLoader loader;
- //Loading default config files core-site.xml and hdfs-site.xml from the config path
- hdfs::optional<hdfs::HdfsConfiguration> config = loader.LoadDefaultResources<hdfs::HdfsConfiguration>();
- //TODO: HDFS-9539 - after this is resolved, valid config will always be returned.
- if(config){
- //Loading options from the config
- options = config->GetOptions();
- }
- //TODO: HDFS-9539 - until then we increase the time-out to allow all recursive async calls to finish
- options.rpc_timeout = std::numeric_limits<int>::max();
- hdfs::IoService * io_service = hdfs::IoService::New();
- //Wrapping fs into a unique pointer to guarantee deletion
- std::shared_ptr<hdfs::FileSystem> fs(hdfs::FileSystem::New(io_service, "", options));
+ //Building a URI object from the given uri path
+ hdfs::URI uri = hdfs::parse_path_or_exit(path);
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, true);
if (!fs) {
- std::cerr << "Could not connect the file system." << std::endl;
- exit(EXIT_FAILURE);
- }
- hdfs::Status status = fs->ConnectToDefaultFs();
- if (!status.ok()) {
- if(!options.defaultFS.get_host().empty()){
- std::cerr << "Error connecting to " << options.defaultFS << ". " << status.ToString() << std::endl;
- } else {
- std::cerr << "Error connecting to the cluster: defaultFS is empty. " << status.ToString() << std::endl;
- }
+ std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/config_parser.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/config_parser.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/config_parser.h
new file mode 100644
index 0000000..e30e2cf
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/config_parser.h
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef LIBHDFSPP_CONFIGPARSER_H_
+#define LIBHDFSPP_CONFIGPARSER_H_
+
+#include "hdfspp/options.h"
+#include "hdfspp/uri.h"
+#include "hdfspp/status.h"
+
+#include <string>
+#include <memory>
+#include <vector>
+
+namespace hdfs {
+
+class ConfigParser {
+ public:
+ ConfigParser();
+ ConfigParser(const std::string& path);
+ ConfigParser(const std::vector<std::string>& configDirectories);
+ ~ConfigParser();
+ ConfigParser(ConfigParser&&);
+ ConfigParser& operator=(ConfigParser&&);
+
+ bool LoadDefaultResources();
+ std::vector<std::pair<std::string, Status> > ValidateResources() const;
+
+ // Return false if value couldn't be found or cast to desired type
+ bool get_int(const std::string& key, int& outval) const;
+ int get_int_or(const std::string& key, const int defaultval) const;
+
+ bool get_string(const std::string& key, std::string& outval) const;
+ std::string get_string_or(const std::string& key, const std::string& defaultval) const;
+
+ bool get_bool(const std::string& key, bool& outval) const;
+ bool get_bool_or(const std::string& key, const bool defaultval) const;
+
+ bool get_double(const std::string& key, double& outval) const;
+ double get_double_or(const std::string& key, const double defaultval) const;
+
+ bool get_uri(const std::string& key, URI& outval) const;
+ URI get_uri_or(const std::string& key, const URI& defaultval) const;
+
+ bool get_options(Options& outval) const;
+ Options get_options_or(const Options& defaultval) const;
+
+ private:
+ class impl;
+ std::unique_ptr<impl> pImpl;
+};
+
+}
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
index 611da21..2fdeec9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
@@ -26,6 +26,7 @@
#include "hdfspp/fsinfo.h"
#include "hdfspp/content_summary.h"
#include "hdfspp/uri.h"
+#include "hdfspp/config_parser.h"
#include "hdfspp/locks.h"
#include <functional>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
index 7e895b1..3e9ee48 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
@@ -1295,13 +1295,13 @@ tOffset hdfsTell(hdfsFS fs, hdfsFile file) {
return -1;
}
- ssize_t offset = 0;
+ off_t offset = 0;
Status stat = file->get_impl()->Seek(&offset, std::ios_base::cur);
if (!stat.ok()) {
return Error(stat);
}
- return offset;
+ return (tOffset)offset;
} catch (const std::exception & e) {
return ReportException(e);
} catch (...) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
index 15e65c1..5d9e52c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
@@ -19,6 +19,6 @@ if(NEED_LINK_DL)
set(LIB_DL dl)
endif()
-add_library(common_obj OBJECT status.cc sasl_digest_md5.cc hdfs_ioservice.cc options.cc configuration.cc configuration_loader.cc hdfs_configuration.cc uri.cc util.cc retry_policy.cc cancel_tracker.cc logging.cc libhdfs_events_impl.cc auth_info.cc namenode_info.cc statinfo.cc fsinfo.cc content_summary.cc locks.cc)
+add_library(common_obj OBJECT status.cc sasl_digest_md5.cc hdfs_ioservice.cc options.cc configuration.cc configuration_loader.cc hdfs_configuration.cc uri.cc util.cc retry_policy.cc cancel_tracker.cc logging.cc libhdfs_events_impl.cc auth_info.cc namenode_info.cc statinfo.cc fsinfo.cc content_summary.cc locks.cc config_parser.cc)
add_library(common $<TARGET_OBJECTS:common_obj> $<TARGET_OBJECTS:uriparser2_obj>)
target_link_libraries(common ${LIB_DL})
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/config_parser.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/config_parser.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/config_parser.cc
new file mode 100644
index 0000000..f7b1c25
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/config_parser.cc
@@ -0,0 +1,219 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hdfspp/config_parser.h"
+#include "common/hdfs_configuration.h"
+#include "common/configuration_loader.h"
+
+#include <string>
+#include <memory>
+#include <vector>
+#include <numeric>
+
+namespace hdfs {
+
+static const char kSearchPathSeparator = ':';
+
+HdfsConfiguration LoadDefault(ConfigurationLoader & loader)
+{
+ optional<HdfsConfiguration> result = loader.LoadDefaultResources<HdfsConfiguration>();
+ if (result)
+ {
+ return result.value();
+ }
+ else
+ {
+ return loader.NewConfig<HdfsConfiguration>();
+ }
+}
+
+class ConfigParser::impl {
+ public:
+ impl() :
+ config_(loader_.NewConfig<HdfsConfiguration>()) {
+ }
+
+ impl(const std::vector<std::string>& dirs) :
+ config_(loader_.NewConfig<HdfsConfiguration>()) {
+
+ // Convert vector of paths into ':' separated path
+ std::string path = std::accumulate(dirs.begin(), dirs.end(), std::string(""),
+ [](std::string cumm, std::string elem) {return cumm + kSearchPathSeparator + elem;});
+ loader_.SetSearchPath(path);
+ config_ = LoadDefault(loader_);
+ }
+
+ impl(const std::string& path) :
+ config_(loader_.NewConfig<HdfsConfiguration>()) {
+
+ loader_.SetSearchPath(path);
+ config_ = LoadDefault(loader_);
+ }
+
+ bool LoadDefaultResources() {
+ config_ = LoadDefault(loader_);
+ return true;
+ }
+
+ std::vector<std::pair<std::string, Status> > ValidateResources() const {
+ return loader_.ValidateDefaultResources<HdfsConfiguration>();
+ }
+
+ bool get_int(const std::string& key, int& outval) const {
+ auto ret = config_.GetInt(key);
+ if (!ret) {
+ return false;
+ } else {
+ outval = *ret;
+ return true;
+ }
+ }
+
+ bool get_string(const std::string& key, std::string& outval) const {
+ auto ret = config_.Get(key);
+ if (!ret) {
+ return false;
+ } else {
+ outval = *ret;
+ return true;
+ }
+ }
+
+ bool get_bool(const std::string& key, bool& outval) const {
+ auto ret = config_.GetBool(key);
+ if (!ret) {
+ return false;
+ } else {
+ outval = *ret;
+ return true;
+ }
+ }
+
+ bool get_double(const std::string& key, double& outval) const {
+ auto ret = config_.GetDouble(key);
+ if (!ret) {
+ return false;
+ } else {
+ outval = *ret;
+ return true;
+ }
+ }
+
+ bool get_uri(const std::string& key, URI& outval) const {
+ auto ret = config_.GetUri(key);
+ if (!ret) {
+ return false;
+ } else {
+ outval = *ret;
+ return true;
+ }
+ }
+
+ bool get_options(Options& outval) {
+ outval = config_.GetOptions();
+ return true;
+ }
+
+ private:
+ ConfigurationLoader loader_;
+ HdfsConfiguration config_;
+};
+
+
+ConfigParser::ConfigParser() {
+ pImpl.reset(new ConfigParser::impl());
+}
+
+ConfigParser::ConfigParser(const std::vector<std::string>& configDirectories) {
+ pImpl.reset(new ConfigParser::impl(configDirectories));
+}
+
+ConfigParser::ConfigParser(const std::string& path) {
+ pImpl.reset(new ConfigParser::impl(path));
+}
+
+ConfigParser::~ConfigParser() = default;
+ConfigParser::ConfigParser(ConfigParser&&) = default;
+ConfigParser& ConfigParser::operator=(ConfigParser&&) = default;
+
+bool ConfigParser::LoadDefaultResources() { return pImpl->LoadDefaultResources(); }
+std::vector<std::pair<std::string, Status> > ConfigParser::ValidateResources() const { return pImpl->ValidateResources();}
+
+bool ConfigParser::get_int(const std::string& key, int& outval) const { return pImpl->get_int(key, outval); }
+int ConfigParser::get_int_or(const std::string& key, const int defaultval) const {
+ int res = 0;
+ if(get_int(key, res)) {
+ return res;
+ } else {
+ return defaultval;
+ }
+}
+
+bool ConfigParser::get_string(const std::string& key, std::string& outval) const { return pImpl->get_string(key, outval); }
+std::string ConfigParser::get_string_or(const std::string& key, const std::string& defaultval) const {
+ std::string res;
+ if(get_string(key, res)) {
+ return res;
+ } else {
+ return defaultval;
+ }
+}
+
+bool ConfigParser::get_bool(const std::string& key, bool& outval) const { return pImpl->get_bool(key, outval); }
+bool ConfigParser::get_bool_or(const std::string& key, const bool defaultval) const {
+ bool res = false;
+ if(get_bool(key, res)) {
+ return res;
+ } else {
+ return defaultval;
+ }
+}
+
+bool ConfigParser::get_double(const std::string& key, double& outval) const { return pImpl->get_double(key, outval); }
+double ConfigParser::get_double_or(const std::string& key, const double defaultval) const {
+ double res = 0;
+ if(get_double(key, res)) {
+ return res;
+ } else {
+ return defaultval;
+ }
+}
+
+bool ConfigParser::get_uri(const std::string& key, URI& outval) const { return pImpl->get_uri(key, outval); }
+URI ConfigParser::get_uri_or(const std::string& key, const URI& defaultval) const {
+ URI res;
+ if(get_uri(key, res)) {
+ return res;
+ } else {
+ res = defaultval;
+ return res;
+ }
+}
+
+bool ConfigParser::get_options(Options& outval) const { return pImpl->get_options(outval); }
+Options ConfigParser::get_options_or(const Options& defaultval) const {
+ Options res;
+ if(get_options(res)) {
+ return res;
+ } else {
+ res = defaultval;
+ return res;
+ }
+}
+
+} // end namespace hdfs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.cc
index 7fbf7c9..298de1e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.cc
@@ -72,8 +72,8 @@ optional<int64_t> Configuration::GetInt(const std::string& key) const {
if (raw) {
errno = 0;
char* end = nullptr;
- auto result =
- std::experimental::make_optional(strtol(raw->c_str(), &end, 10));
+ optional<int64_t> result =
+ std::experimental::make_optional(static_cast<int64_t>(strtol(raw->c_str(), &end, 10)));
if (end == raw->c_str()) {
/* strtoll will set end to input if no conversion was done */
return optional<int64_t>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.cc
index e1434dad..691d2ff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.cc
@@ -17,6 +17,7 @@
*/
#include "configuration_loader.h"
+#include "common/logging.h"
#include <fstream>
#include <strings.h>
@@ -74,7 +75,11 @@ void ConfigurationLoader::SetDefaultSearchPath() {
// /etc/hadoop/conf
const char * hadoop_conf_dir_env = getenv("HADOOP_CONF_DIR");
if (hadoop_conf_dir_env) {
- AddToSearchPath(hadoop_conf_dir_env);
+ std::stringstream ss(hadoop_conf_dir_env);
+ std::string path;
+ while (std::getline(ss, path, kSearchPathSeparator)) {
+ AddToSearchPath(path);
+ }
} else {
AddToSearchPath("/etc/hadoop/conf");
}
@@ -136,6 +141,63 @@ std::string ConfigurationLoader::GetSearchPath()
return result.str();
}
+Status validateStream(std::istream & stream) {
+ std::streampos start = stream.tellg();
+ stream.seekg(0, std::ios::end);
+ std::streampos end = stream.tellg();
+ stream.seekg(start, std::ios::beg);
+
+ int length = end - start;
+
+ if (length <= 0 || start == -1 || end == -1)
+ return Status::Error("The configuration file is empty");
+
+ LOG_DEBUG(kFileSystem, << "validateStream will read a config file of length " << length);
+
+ std::vector<char> raw_bytes((int64_t)length + 1);
+ stream.read(&raw_bytes[0], length);
+ raw_bytes[length] = 0;
+
+ try {
+ rapidxml::xml_document<> dom;
+ dom.parse<rapidxml::parse_trim_whitespace|rapidxml::parse_validate_closing_tags>(&raw_bytes[0]);
+
+ /* File must contain a single <configuration> stanza */
+ auto config_node = dom.first_node("configuration", 0, false);
+ if (!config_node) {
+ return Status::Error("The configuration file is missing a 'configuration' tag");
+ }
+ return Status::OK();
+ } catch (const rapidxml::parse_error &e) {
+ size_t location = e.where<char>() - &raw_bytes[0];
+ std::string msg = "The configuration file has invalid xml around character " + std::to_string(location);
+ return Status::Error(msg.c_str());
+ }
+}
+
+std::vector<std::pair<std::string, Status> > ConfigurationLoader::ValidateResources(std::vector<std::string> filenames) const
+{
+ std::vector<std::pair<std::string, Status> > stats;
+ bool found;
+ for(auto file: filenames) {
+ found = false;
+ for(auto dir: search_path_) {
+ std::ifstream stream(dir + file);
+ if ( stream.is_open() ) {
+ found = true;
+ stats.push_back(std::make_pair(file,validateStream(stream)));
+ } else {
+ LOG_DEBUG(kFileSystem, << dir << file << " was not found");
+ }
+ }
+ if(!found) {
+ std::string msg("No directory in the current search path contains the file [" + file + "]");
+ stats.push_back(std::make_pair(file,Status::PathNotFound(msg.c_str())));
+ }
+ }
+ return stats;
+}
+
bool ConfigurationLoader::UpdateMapWithFile(ConfigMap & map, const std::string & path) const
{
if (path.front() == kFileSeparator) { // Absolute path
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.h
index 51ac23a..2673a6b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.h
@@ -20,6 +20,7 @@
#define COMMON_CONFIGURATION_BUILDER_H_
#include "configuration.h"
+#include "hdfspp/status.h"
namespace hdfs {
@@ -75,6 +76,13 @@ public:
optional<T> LoadDefaultResources();
+ // Returns a vector of filenames and the corresponding status when validation is attempted.
+ // If the files can be successfully validated, then the status returned for that file is Status::OK
+ // The files that are validated are those returned by T::GetDefaultFilenames().
+ // T must be Configuration or a subclass
+ template<class T>
+ std::vector<std::pair<std::string, Status>> ValidateDefaultResources() const;
+
/****************************************************************************
* SEARCH PATH METHODS
***************************************************************************/
@@ -98,6 +106,8 @@ public:
protected:
using ConfigMap = Configuration::ConfigMap;
+ std::vector<std::pair<std::string, Status>> ValidateResources(std::vector<std::string> filenames) const;
+
// Updates the src map with data from the XML in the path
// The search path will be searched for the filename
bool UpdateMapWithFile(ConfigMap & map, const std::string & path) const;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader_impl.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader_impl.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader_impl.h
index 6258450..dad5a82 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader_impl.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader_impl.h
@@ -111,6 +111,11 @@ optional<T> ConfigurationLoader::LoadDefaultResources() {
}
}
+template<class T>
+std::vector<std::pair<std::string, Status> > ConfigurationLoader::ValidateDefaultResources() const{
+ return ValidateResources(T::GetDefaultFilenames());
+}
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/continuation/protobuf.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/continuation/protobuf.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/continuation/protobuf.h
index 3dc75c2..21e063e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/continuation/protobuf.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/continuation/protobuf.h
@@ -49,6 +49,7 @@ struct ReadDelimitedPBMessageContinuation : public Continuation {
uint32_t size = 0;
bool v = is.ReadVarint32(&size);
assert(v);
+ (void)v; //avoids unused variable warning
is.PushLimit(size);
msg_->Clear();
v = msg_->MergeFromCodedStream(&is);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.cc
index 148c674..dca49fb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.cc
@@ -54,6 +54,7 @@ RetryAction FixedDelayWithFailover::ShouldRetry(const Status &s, uint64_t retrie
uint64_t failovers,
bool isIdempotentOrAtMostOnce) const {
(void)isIdempotentOrAtMostOnce;
+ (void)max_failover_conn_retries_;
LOG_TRACE(kRPC, << "FixedDelayWithFailover::ShouldRetry(retries=" << retries << ", failovers=" << failovers << ")");
if(failovers < max_failover_retries_ && (s.code() == ::asio::error::timed_out || s.get_server_exception_type() == Status::kStandbyException) )
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
index 2087d53..ba702b0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
@@ -52,7 +52,7 @@ void FileHandleImpl::PositionRead(
const std::function<void(const Status &, size_t)> &handler) {
LOG_DEBUG(kFileHandle, << "FileHandleImpl::PositionRead("
<< FMT_THIS_ADDR << ", buf=" << buf
- << ", buf_size=" << buf_size << ") called");
+ << ", buf_size=" << std::to_string(buf_size) << ") called");
/* prevent usage after cancelation */
if(cancel_state_->is_canceled()) {
@@ -78,7 +78,7 @@ void FileHandleImpl::PositionRead(
Status FileHandleImpl::PositionRead(void *buf, size_t buf_size, off_t offset, size_t *bytes_read) {
LOG_DEBUG(kFileHandle, << "FileHandleImpl::[sync]PositionRead("
<< FMT_THIS_ADDR << ", buf=" << buf
- << ", buf_size=" << buf_size
+ << ", buf_size=" << std::to_string(buf_size)
<< ", offset=" << offset << ") called");
auto callstate = std::make_shared<std::promise<std::tuple<Status, size_t>>>();
@@ -106,7 +106,7 @@ Status FileHandleImpl::PositionRead(void *buf, size_t buf_size, off_t offset, si
Status FileHandleImpl::Read(void *buf, size_t buf_size, size_t *bytes_read) {
LOG_DEBUG(kFileHandle, << "FileHandleImpl::Read("
<< FMT_THIS_ADDR << ", buf=" << buf
- << ", buf_size=" << buf_size << ") called");
+ << ", buf_size=" << std::to_string(buf_size) << ") called");
Status stat = PositionRead(buf, buf_size, offset_, bytes_read);
if(!stat.ok()) {
@@ -237,7 +237,7 @@ void FileHandleImpl::AsyncPreadSome(
LOG_DEBUG(kFileHandle, << "FileHandleImpl::AsyncPreadSome("
<< FMT_THIS_ADDR << "), ...) Datanode hostname=" << dnHostName << ", IP Address=" << dnIpAddr
- << ", file path=\"" << path_ << "\", offset=" << offset << ", read size=" << size_within_block);
+ << ", file path=\"" << path_ << "\", offset=" << std::to_string(offset) << ", read size=" << size_within_block);
// This is where we will put the logic for re-using a DN connection; we can
// steal the FileHandle's dn and put it back when we're done
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
index d97eb25..ac6c3d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
@@ -203,9 +203,9 @@ public:
/* all monitored events will need to lookup handlers */
std::shared_ptr<LibhdfsEvents> get_event_handlers();
- Options get_options();
+ Options get_options() override;
- std::string get_cluster_name();
+ std::string get_cluster_name() override;
private:
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
index e289d5e..97c0e18 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
@@ -41,7 +41,7 @@ namespace hdfs {
class NameNodeOperations {
public:
- MEMCHECKED_CLASS(NameNodeOperations);
+ MEMCHECKED_CLASS(NameNodeOperations)
NameNodeOperations(::asio::io_service *io_service, const Options &options,
const std::string &client_name, const std::string &user_name,
const char *protocol_name, int protocol_version) :
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc
index 877911c..ca7715d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc
@@ -177,6 +177,7 @@ struct BlockReaderImpl::ReadPacketHeader : continuation::Continuation
bool v = parent_->header_.ParseFromArray(&buf_[kHeaderStart],
header_length());
assert(v && "Failed to parse the header");
+ (void)v; //avoids unused variable warning
parent_->state_ = kReadChecksum;
}
if(parent_->event_handlers_) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
index 06b1616..947d121 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
@@ -31,8 +31,6 @@ namespace pbio = ::google::protobuf::io;
using namespace ::hadoop::common;
using namespace ::std::placeholders;
-static const int kNoRetry = -1;
-
static void AddHeadersToPacket(
std::string *res, std::initializer_list<const pb::MessageLite *> headers,
const std::string *payload) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h
index 4339a48..678dab3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h
@@ -38,14 +38,14 @@ namespace hdfs {
template <class Socket>
class RpcConnectionImpl : public RpcConnection {
public:
- MEMCHECKED_CLASS(RpcConnectionImpl);
+ MEMCHECKED_CLASS(RpcConnectionImpl)
RpcConnectionImpl(RpcEngine *engine);
virtual ~RpcConnectionImpl() override;
virtual void Connect(const std::vector<::asio::ip::tcp::endpoint> &server,
const AuthInfo & auth_info,
- RpcCallback &handler);
+ RpcCallback &handler) override;
virtual void ConnectAndFlush(
const std::vector<::asio::ip::tcp::endpoint> &server) override;
virtual void SendHandshake(RpcCallback &handler) override;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
index 04ed7eb..f59f2ce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
@@ -68,7 +68,7 @@ void RpcEngine::Connect(const std::string &cluster_name,
}
// Construct retry policy after we determine if config is HA
- retry_policy_ = std::move(MakeRetryPolicy(options_));
+ retry_policy_ = MakeRetryPolicy(options_);
conn_ = InitializeConnection();
conn_->Connect(last_endpoints_, auth_info_, handler);
@@ -133,7 +133,7 @@ std::string RpcEngine::getRandomClientId()
void RpcEngine::TEST_SetRpcConnection(std::shared_ptr<RpcConnection> conn) {
conn_ = conn;
- retry_policy_ = std::move(MakeRetryPolicy(options_));
+ retry_policy_ = MakeRetryPolicy(options_);
}
void RpcEngine::TEST_SetRetryPolicy(std::unique_ptr<const RetryPolicy> policy) {
@@ -188,7 +188,7 @@ void RpcEngine::AsyncRpcCommsError(
const Status &status,
std::shared_ptr<RpcConnection> failedConnection,
std::vector<std::shared_ptr<Request>> pendingRequests) {
- LOG_ERROR(kRPC, << "RpcEngine::AsyncRpcCommsError called; status=\"" << status.ToString() << "\" conn=" << failedConnection.get() << " reqs=" << pendingRequests.size());
+ LOG_ERROR(kRPC, << "RpcEngine::AsyncRpcCommsError called; status=\"" << status.ToString() << "\" conn=" << failedConnection.get() << " reqs=" << std::to_string(pendingRequests.size()));
io_service().post([this, status, failedConnection, pendingRequests]() {
RpcCommsError(status, failedConnection, pendingRequests);
@@ -199,7 +199,7 @@ void RpcEngine::RpcCommsError(
const Status &status,
std::shared_ptr<RpcConnection> failedConnection,
std::vector<std::shared_ptr<Request>> pendingRequests) {
- LOG_WARN(kRPC, << "RpcEngine::RpcCommsError called; status=\"" << status.ToString() << "\" conn=" << failedConnection.get() << " reqs=" << pendingRequests.size());
+ LOG_WARN(kRPC, << "RpcEngine::RpcCommsError called; status=\"" << status.ToString() << "\" conn=" << failedConnection.get() << " reqs=" << std::to_string(pendingRequests.size()));
std::lock_guard<std::mutex> state_lock(engine_state_lock_);
@@ -252,7 +252,7 @@ void RpcEngine::RpcCommsError(
head_action && head_action->action != RetryAction::FAIL;
if (haveRequests) {
- LOG_TRACE(kRPC, << "Have " << pendingRequests.size() << " requests to resend");
+ LOG_TRACE(kRPC, << "Have " << std::to_string(pendingRequests.size()) << " requests to resend");
bool needNewConnection = !conn_;
if (needNewConnection) {
LOG_DEBUG(kRPC, << "Creating a new NN conection");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_engine.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_engine.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_engine.cc
index 3cebe7e..c5b90f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_engine.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_engine.cc
@@ -53,7 +53,7 @@ Status SaslEngine::SetPasswordInfo(const std::string &id,
bool SaslEngine::ChooseMech(const std::vector<SaslMethod> &resp_auths) {
Status status = Status::OK();
- if (resp_auths.empty()) return NULL;
+ if (resp_auths.empty()) return false;
for (SaslMethod auth: resp_auths) {
if ( auth.mechanism != "GSSAPI") continue; // Hack: only GSSAPI for now
@@ -74,7 +74,7 @@ bool SaslEngine::ChooseMech(const std::vector<SaslMethod> &resp_auths) {
// Clear out the chosen mech
chosen_mech_ = SaslMethod();
- return NULL;
+ return false;
} // choose_mech()
} // namespace hdfs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.h
index 7947ff5..9ad11b7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.h
@@ -18,6 +18,7 @@
#ifndef TESTS_CONFIGURATION_H_
#define TESTS_CONFIGURATION_H_
+#include "hdfspp/config_parser.h"
#include "common/configuration.h"
#include "common/configuration_loader.h"
#include <cstdio>
@@ -50,6 +51,28 @@ void simpleConfigStream(std::stringstream& out, Args... args) {
out << "</configuration>";
}
+template <typename T, typename U>
+void damagedConfigStreamProperty(std::stringstream& out, T key, U value) {
+ out << "<propertyy>"
+ << "<name>" << key << "</name>"
+ << "<value>" << value << "</value>"
+ << "</property>";
+}
+
+template <typename T, typename U, typename... Args>
+void damagedConfigStreamProperty(std::stringstream& out, T key, U value,
+ Args... args) {
+ damagedConfigStreamProperty(out, key, value);
+ damagedConfigStreamProperty(out, args...);
+}
+
+template <typename... Args>
+void damagedConfigStream(std::stringstream& out, Args... args) {
+ out << "<configuration>";
+ damagedConfigStreamProperty(out, args...);
+ out << "</configuration>";
+}
+
template <typename... Args>
optional<Configuration> simpleConfig(Args... args) {
std::stringstream stream;
@@ -72,6 +95,16 @@ void writeSimpleConfig(const std::string& filename, Args... args) {
out << stream.rdbuf();
}
+template <typename... Args>
+void writeDamagedConfig(const std::string& filename, Args... args) {
+ std::stringstream stream;
+ damagedConfigStream(stream, args...);
+
+ std::ofstream out;
+ out.open(filename);
+ out << stream.rdbuf();
+}
+
// TempDir: is deleted on destruction
class TempFile {
public:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_configuration_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_configuration_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_configuration_test.cc
index 360f886..b21725c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_configuration_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_configuration_test.cc
@@ -19,6 +19,7 @@
#include "common/hdfs_configuration.h"
#include "configuration_test.h"
#include <gmock/gmock.h>
+#include <iostream>
using ::testing::_;
@@ -111,6 +112,53 @@ TEST(HdfsConfigurationTest, TestDefaultConfigs) {
EXPECT_TRUE(config && "Parse streams");
EXPECT_EQ("value2", config->GetWithDefault("key2", ""));
}
+
+
+}
+
+TEST(HdfsConfigurationTest, TestConfigParserAPI) {
+ // Config parser API
+ {
+ TempDir tempDir;
+ TempFile coreSite(tempDir.path + "/core-site.xml");
+ writeSimpleConfig(coreSite.filename, "key1", "value1");
+ TempFile hdfsSite(tempDir.path + "/hdfs-site.xml");
+ writeSimpleConfig(hdfsSite.filename, "key2", "value2");
+
+ ConfigParser parser(tempDir.path);
+
+ EXPECT_EQ("value1", parser.get_string_or("key1", ""));
+ EXPECT_EQ("value2", parser.get_string_or("key2", ""));
+
+ auto stats = parser.ValidateResources();
+
+ EXPECT_EQ("core-site.xml", stats[0].first);
+ EXPECT_EQ("OK", stats[0].second.ToString());
+
+ EXPECT_EQ("hdfs-site.xml", stats[1].first);
+ EXPECT_EQ("OK", stats[1].second.ToString());
+ }
+
+ {
+ TempDir tempDir;
+ TempFile coreSite(tempDir.path + "/core-site.xml");
+ writeSimpleConfig(coreSite.filename, "key1", "value1");
+ TempFile hdfsSite(tempDir.path + "/hdfs-site.xml");
+ writeDamagedConfig(hdfsSite.filename, "key2", "value2");
+
+ ConfigParser parser(tempDir.path);
+
+ EXPECT_EQ("value1", parser.get_string_or("key1", ""));
+ EXPECT_EQ("", parser.get_string_or("key2", ""));
+
+ auto stats = parser.ValidateResources();
+
+ EXPECT_EQ("core-site.xml", stats[0].first);
+ EXPECT_EQ("OK", stats[0].second.ToString());
+
+ EXPECT_EQ("hdfs-site.xml", stats[1].first);
+ EXPECT_EQ("Exception:The configuration file has invalid xml around character 74", stats[1].second.ToString());
+ }
}
int main(int argc, char *argv[])
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc
index 5b0cf4e..6cc5a5b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc
@@ -22,15 +22,29 @@
namespace hdfs {
std::shared_ptr<hdfs::FileSystem> doConnect(hdfs::URI & uri, bool max_timeout) {
+
+ //This sets the config path to the default: "$HADOOP_CONF_DIR" or "/etc/hadoop/conf"
+ //and loads default config files core-site.xml and hdfs-site.xml from the config path
+ hdfs::ConfigParser parser;
+ if(!parser.LoadDefaultResources()){
+ std::cerr << "Could not load default resources. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ auto stats = parser.ValidateResources();
+ //validating core-site.xml
+ if(!stats[0].second.ok()){
+ std::cerr << stats[0].first << " is invalid: " << stats[0].second.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ //validating hdfs-site.xml
+ if(!stats[1].second.ok()){
+ std::cerr << stats[1].first << " is invalid: " << stats[1].second.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
hdfs::Options options;
- //Setting the config path to the default: "$HADOOP_CONF_DIR" or "/etc/hadoop/conf"
- hdfs::ConfigurationLoader loader;
- //Loading default config files core-site.xml and hdfs-site.xml from the config path
- hdfs::optional<HdfsConfiguration> config = loader.LoadDefaultResources<HdfsConfiguration>();
- //TODO: HDFS-9539 - after this is resolved, valid config will always be returned.
- if(config){
- //Loading options from the config
- options = config->GetOptions();
+ if(!parser.get_options(options)){
+ std::cerr << "Could not load Options object. " << std::endl;
+ exit(EXIT_FAILURE);
}
if(max_timeout){
//TODO: HDFS-9539 - until then we increase the time-out to allow all recursive async calls to finish
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12942f67/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.h
index 7e725e4..6e02e14 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.h
@@ -21,9 +21,6 @@
#define TOOLS_COMMON_H_
#include "hdfspp/hdfspp.h"
-#include "hdfspp/uri.h"
-#include "common/hdfs_configuration.h"
-#include "common/configuration_loader.h"
#include <mutex>
namespace hdfs {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/52] [abbrv] hadoop git commit: HDFS-12134: libhdfs++: Add a
synchronization interface for the GSSAPI. Contributed by James Clampffer.
Posted by jh...@apache.org.
HDFS-12134: libhdfs++: Add a synchronization interface for the GSSAPI. Contributed by James Clampffer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/22ea06a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/22ea06a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/22ea06a3
Branch: refs/heads/HDFS-8707
Commit: 22ea06a3dd74b80737977908e30ec38a32b968ab
Parents: 033433b
Author: James Clampffer <ja...@hp.com>
Authored: Mon Aug 7 13:04:50 2017 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../native/libhdfspp/include/hdfspp/hdfspp.h | 1 +
.../native/libhdfspp/include/hdfspp/locks.h | 110 +++++++++
.../native/libhdfspp/include/hdfspp/status.h | 2 +
.../native/libhdfspp/lib/common/CMakeLists.txt | 2 +-
.../main/native/libhdfspp/lib/common/locks.cc | 100 +++++++++
.../main/native/libhdfspp/lib/common/status.cc | 11 +-
.../src/main/native/libhdfspp/lib/common/uri.cc | 1 +
.../libhdfspp/lib/rpc/cyrus_sasl_engine.cc | 100 +++++++--
.../native/libhdfspp/lib/rpc/gsasl_engine.cc | 110 ++++++---
.../native/libhdfspp/lib/rpc/sasl_protocol.cc | 2 -
.../main/native/libhdfspp/tests/CMakeLists.txt | 3 +
.../src/main/native/libhdfspp/tests/uri_test.cc | 1 -
.../native/libhdfspp/tests/user_lock_test.cc | 225 +++++++++++++++++++
13 files changed, 610 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22ea06a3/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
index 673455e..611da21 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
@@ -26,6 +26,7 @@
#include "hdfspp/fsinfo.h"
#include "hdfspp/content_summary.h"
#include "hdfspp/uri.h"
+#include "hdfspp/locks.h"
#include <functional>
#include <memory>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22ea06a3/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/locks.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/locks.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/locks.h
new file mode 100644
index 0000000..3dfeab4
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/locks.h
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef COMMON_HDFS_LOCKS_H_
+#define COMMON_HDFS_LOCKS_H_
+
+#include <stdexcept>
+#include <string>
+#include <atomic>
+#include <mutex>
+#include <memory>
+
+namespace hdfs
+{
+
+//
+// Thrown by LockGuard to indicate that it was unable to acquire a mutex
+// what_str should contain info about what caused the failure
+//
+class LockFailure : public std::runtime_error {
+ public:
+ LockFailure(const char *what_str) : std::runtime_error(what_str) {};
+ LockFailure(const std::string& what_str) : std::runtime_error(what_str) {};
+};
+
+//
+// A pluggable mutex type to allow client code to share mutexes it may
+// already use to protect certain system resources. Certain shared
+// libraries have some procedures that aren't always implemented in a thread
+// safe manner. If libhdfs++ and the code linking it depend on the same
+// library this provides a mechanism to coordinate safe access.
+//
+// Interface provided is intended to be similar to std::mutex. If the lock
+// can't be aquired it may throw LockFailure from the lock method. If lock
+// does fail libhdfs++ is expected fail as cleanly as possible e.g.
+// FileSystem::Mkdirs might return a MutexError but a subsequent call may be
+// successful.
+//
+class Mutex {
+ public:
+ virtual ~Mutex() {};
+ virtual void lock() = 0;
+ virtual void unlock() = 0;
+ virtual std::string str() = 0;
+};
+
+//
+// LockGuard works in a similar manner to std::lock_guard: it locks the mutex
+// in the constructor and unlocks it in the destructor.
+// Failure to acquire the mutex in the constructor will result in throwing a
+// LockFailure exception.
+//
+class LockGuard {
+ public:
+ LockGuard(Mutex *m);
+ ~LockGuard();
+ private:
+ Mutex *_mtx;
+};
+
+//
+// Manage instances of hdfs::Mutex that are intended to be global to the
+// process.
+//
+// LockManager's InitLocks method provides a mechanism for the calling
+// application to share its own implementations of hdfs::Mutex. It must be
+// called prior to instantiating any FileSystem objects and can only be
+// called once. If a lock is not provided a default mutex type wrapping
+// std::mutex is used as a default.
+//
+
+class LockManager {
+ public:
+ // Initializes with a default set of C++11 style mutexes
+ static bool InitLocks(Mutex *gssapi);
+ static Mutex *getGssapiMutex();
+
+ // Tests only, implementation may no-op on release builds.
+ // Reset _finalized to false and set all Mutex* members to default values.
+ static void TEST_reset_manager();
+ static Mutex *TEST_get_default_mutex();
+ private:
+ // Used only in tests.
+ static Mutex *TEST_default_mutex;
+ // Use to synchronize calls into GSSAPI/Kerberos libs
+ static Mutex *gssapiMtx;
+
+ // Prevent InitLocks from being called more than once
+ // Allows all locks to be set a single time atomically
+ static std::mutex _state_lock;
+ static bool _finalized;
+};
+
+} // end namespace hdfs
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22ea06a3/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
index d0922ae..718e530 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
@@ -50,6 +50,7 @@ class Status {
static Status PathNotFound(const char *msg);
static Status InvalidOffset(const char *msg);
static Status PathIsNotDirectory(const char *msg);
+ static Status MutexError(const char *msg);
// success
bool ok() const { return code_ == 0; }
@@ -79,6 +80,7 @@ class Status {
kNotADirectory = static_cast<unsigned>(std::errc::not_a_directory),
kFileAlreadyExists = static_cast<unsigned>(std::errc::file_exists),
kPathIsNotEmptyDirectory = static_cast<unsigned>(std::errc::directory_not_empty),
+ kBusy = static_cast<unsigned>(std::errc::device_or_resource_busy),
// non-errc codes start at 256
kException = 256,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22ea06a3/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
index b0b721a..15e65c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
@@ -19,6 +19,6 @@ if(NEED_LINK_DL)
set(LIB_DL dl)
endif()
-add_library(common_obj OBJECT status.cc sasl_digest_md5.cc hdfs_ioservice.cc options.cc configuration.cc configuration_loader.cc hdfs_configuration.cc uri.cc util.cc retry_policy.cc cancel_tracker.cc logging.cc libhdfs_events_impl.cc auth_info.cc namenode_info.cc statinfo.cc fsinfo.cc content_summary.cc)
+add_library(common_obj OBJECT status.cc sasl_digest_md5.cc hdfs_ioservice.cc options.cc configuration.cc configuration_loader.cc hdfs_configuration.cc uri.cc util.cc retry_policy.cc cancel_tracker.cc logging.cc libhdfs_events_impl.cc auth_info.cc namenode_info.cc statinfo.cc fsinfo.cc content_summary.cc locks.cc)
add_library(common $<TARGET_OBJECTS:common_obj> $<TARGET_OBJECTS:uriparser2_obj>)
target_link_libraries(common ${LIB_DL})
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22ea06a3/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/locks.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/locks.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/locks.cc
new file mode 100644
index 0000000..30dcb44
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/locks.cc
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hdfspp/locks.h"
+
+#include <mutex>
+
+
+namespace hdfs {
+
+LockGuard::LockGuard(Mutex *m) : _mtx(m) {
+ if(!m) {
+ throw LockFailure("LockGuard passed invalid (null) Mutex pointer");
+ }
+ _mtx->lock();
+}
+
+LockGuard::~LockGuard() {
+ if(_mtx) {
+ _mtx->unlock();
+ }
+}
+
+
+// Basic mutexes to use as default. Just a wrapper around C++11 std::mutex.
+class DefaultMutex : public Mutex {
+ public:
+ DefaultMutex() {}
+
+ void lock() override {
+ // Could throw in here if the implementation couldn't lock for some reason.
+ _mtx.lock();
+ }
+
+ void unlock() override {
+ _mtx.unlock();
+ }
+
+ std::string str() override {
+ return "DefaultMutex";
+ }
+ private:
+ std::mutex _mtx;
+};
+
+DefaultMutex defaultTestMutex;
+DefaultMutex defaultGssapiMutex;
+
+// LockManager static var instantiation
+Mutex *LockManager::TEST_default_mutex = &defaultTestMutex;
+Mutex *LockManager::gssapiMtx = &defaultGssapiMutex;
+std::mutex LockManager::_state_lock;
+bool LockManager::_finalized = false;
+
+bool LockManager::InitLocks(Mutex *gssapi) {
+ std::lock_guard<std::mutex> guard(_state_lock);
+
+ // You get once shot to set this - swapping the locks
+ // out while in use gets risky. It can still be done by
+ // using the Mutex as a proxy object if one understands
+ // the implied risk of doing so.
+ if(_finalized)
+ return false;
+
+ gssapiMtx = gssapi;
+ _finalized = true;
+ return true;
+}
+
+Mutex *LockManager::getGssapiMutex() {
+ std::lock_guard<std::mutex> guard(_state_lock);
+ return gssapiMtx;
+}
+
+Mutex *LockManager::TEST_get_default_mutex() {
+ return TEST_default_mutex;
+}
+
+void LockManager::TEST_reset_manager() {
+ _finalized = false;
+ // user still responsible for cleanup
+ gssapiMtx = &defaultGssapiMutex;
+}
+
+} // end namepace hdfs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22ea06a3/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
index 5903553..4c5c7be 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
@@ -177,7 +177,16 @@ std::string Status::ToString() const {
}
bool Status::notWorthRetry() const {
- return noRetryExceptions.find(code_) != noRetryExceptions.end();
+ return noRetryExceptions.find(code_) != noRetryExceptions.end();
+}
+
+Status Status::MutexError(const char *msg) {
+ std::string formatted = "MutexError";
+ if(msg) {
+ formatted += ": ";
+ formatted += msg;
+ }
+ return Status(kBusy/*try_lock failure errno*/, msg);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22ea06a3/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.cc
index 2213f8b..9e9319b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.cc
@@ -244,6 +244,7 @@ URI URI::parse_from_string(const std::string &str)
///////////////////////////////////////////////////////////////////////////////
URI::URI() : _port(-1) {}
+
URI::Query::Query(const std::string& k, const std::string& v) : key(k), value(v) {}
std::string URI::str(bool encoded_output) const
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22ea06a3/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/cyrus_sasl_engine.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/cyrus_sasl_engine.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/cyrus_sasl_engine.cc
index 69b2267..5c96ede 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/cyrus_sasl_engine.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/cyrus_sasl_engine.cc
@@ -16,6 +16,8 @@
* limitations under the License.
*/
+#include "hdfspp/locks.h"
+
#include <sys/types.h>
#include "sasl/sasl.h"
#include "sasl/saslutil.h"
@@ -31,6 +33,9 @@
namespace hdfs {
+static Mutex *getSaslMutex() {
+ return LockManager::getGssapiMutex();
+}
// Forward decls of sasl callback functions
typedef int (*sasl_callback_ft)(void);
@@ -111,23 +116,30 @@ Status CySaslEngine::SaslError( int rc) {
* Cyrus SASL ENGINE
*/
- CySaslEngine::CySaslEngine() : SaslEngine(), conn_(nullptr)
- {
- // Create an array of callbacks that embed a pointer to this
- // so we can call methods of the engine
- per_connection_callbacks_ = {
- { SASL_CB_USER, (sasl_callback_ft) & get_name, this}, // userid for authZ
- { SASL_CB_AUTHNAME, (sasl_callback_ft) & get_name, this}, // authid for authT
- { SASL_CB_GETREALM, (sasl_callback_ft) & getrealm, this}, // krb/gssapi realm
- // { SASL_CB_PASS, (sasl_callback_ft)&getsecret, this
- { SASL_CB_LIST_END, (sasl_callback_ft) NULL, NULL}
- };
- }
+CySaslEngine::CySaslEngine() : SaslEngine(), conn_(nullptr)
+{
+ // Create an array of callbacks that embed a pointer to this
+ // so we can call methods of the engine
+ per_connection_callbacks_ = {
+ { SASL_CB_USER, (sasl_callback_ft) & get_name, this}, // userid for authZ
+ { SASL_CB_AUTHNAME, (sasl_callback_ft) & get_name, this}, // authid for authT
+ { SASL_CB_GETREALM, (sasl_callback_ft) & getrealm, this}, // krb/gssapi realm
+ // { SASL_CB_PASS, (sasl_callback_ft)&getsecret, this
+ { SASL_CB_LIST_END, (sasl_callback_ft) NULL, NULL}
+ };
+}
+// Cleanup of last resort. Call Finish to allow a safer check on disposal
CySaslEngine::~CySaslEngine()
{
+
if (conn_) {
+ try {
+ LockGuard saslGuard(getSaslMutex());
sasl_dispose( &conn_); // undo sasl_client_new()
+ } catch (const LockFailure& e) {
+ LOG_ERROR(kRPC, << "Unable to dispose of SASL context due to " << e.what());
+ }
}
} // destructor
@@ -146,8 +158,15 @@ Status CySaslEngine::InitCyrusSasl()
const char * fqdn = chosen_mech_.serverid.c_str();
const char * proto = chosen_mech_.protocol.c_str();
- rc = sasl_client_new(proto, fqdn, NULL, NULL, &per_connection_callbacks_[0], 0, &conn_);
- if (rc != SASL_OK) return SaslError(rc);
+ try {
+ LockGuard saslGuard(getSaslMutex());
+ rc = sasl_client_new(proto, fqdn, NULL, NULL, &per_connection_callbacks_[0], 0, &conn_);
+ if (rc != SASL_OK) {
+ return SaslError(rc);
+ }
+ } catch (const LockFailure& e) {
+ return Status::MutexError("mutex that guards sasl_client_new unable to lock");
+ }
return Status::OK();
} // cysasl_new()
@@ -176,8 +195,15 @@ CySaslEngine::Start()
const char * chosen_mech;
std::string token;
- rc = sasl_client_start(conn_, chosen_mech_.mechanism.c_str(), &client_interact,
- (const char **) &buf, &buflen, &chosen_mech);
+ try {
+ LockGuard saslGuard(getSaslMutex());
+ rc = sasl_client_start(conn_, chosen_mech_.mechanism.c_str(), &client_interact,
+ (const char **) &buf, &buflen, &chosen_mech);
+ } catch (const LockFailure& e) {
+ state_ = kFailure;
+ return std::make_pair( Status::MutexError("mutex that guards sasl_client_new unable to lock"), "" );
+ }
+
switch (rc) {
case SASL_OK: state_ = kSuccess;
@@ -192,6 +218,7 @@ CySaslEngine::Start()
// Cyrus will free this buffer when the connection is shut down
token = std::string( buf, buflen);
return std::make_pair( Status::OK(), token);
+
} // start() method
std::pair<Status, std::string> CySaslEngine::Step(const std::string data)
@@ -203,9 +230,15 @@ std::pair<Status, std::string> CySaslEngine::Step(const std::string data)
if (state_ != kWaitingForData)
LOG_WARN(kRPC, << "CySaslEngine::step when state is " << state_);
- int rc = sasl_client_step(conn_, data.c_str(), data.size(), &client_interact,
- (const char **) &output, &outlen);
-
+ int rc = 0;
+ try {
+ LockGuard saslGuard(getSaslMutex());
+ rc = sasl_client_step(conn_, data.c_str(), data.size(), &client_interact,
+ (const char **) &output, &outlen);
+ } catch (const LockFailure& e) {
+ state_ = kFailure;
+ return std::make_pair( Status::MutexError("mutex that guards sasl_client_new unable to lock"), "" );
+ }
// right now, state_ == kWaitingForData,
// so update state_, to reflect _step()'s result:
switch (rc) {
@@ -224,8 +257,13 @@ Status CySaslEngine::Finish()
LOG_WARN(kRPC, << "CySaslEngine::finish when state is " << state_);
if (conn_ != nullptr) {
+ try {
+ LockGuard saslGuard(getSaslMutex());
sasl_dispose( &conn_);
conn_ = NULL;
+ } catch (const LockFailure& e) {
+ return Status::MutexError("mutex that guards sasl_dispose unable to lock");
+ }
}
return Status::OK();
@@ -234,6 +272,8 @@ Status CySaslEngine::Finish()
//////////////////////////////////////////////////
// Internal callbacks, for sasl_init_client(). //
// Mostly lifted from cyrus' sample_client.c . //
+// Implicitly called in a context that already //
+// holds the SASL/GSSAPI lock. //
//////////////////////////////////////////////////
static int
@@ -388,14 +428,26 @@ const sasl_callback_t per_process_callbacks[] = {
CyrusPerProcessData::CyrusPerProcessData()
{
- int init_rc = sasl_client_init(per_process_callbacks);
- init_status_ = make_status(init_rc);
+ try {
+ LockGuard saslGuard(getSaslMutex());
+ int init_rc = sasl_client_init(per_process_callbacks);
+ init_status_ = make_status(init_rc);
+ } catch (const LockFailure& e) {
+ init_status_ = Status::MutexError("mutex protecting process-wide sasl_client_init unable to lock");
+ }
}
CyrusPerProcessData::~CyrusPerProcessData()
{
// Undo sasl_client_init())
- sasl_done();
+ try {
+ LockGuard saslGuard(getSaslMutex());
+ sasl_done();
+ } catch (const LockFailure& e) {
+ // Not can be done at this point, but the process is most likely shutting down anyway.
+ LOG_ERROR(kRPC, << "mutex protecting process-wide sasl_done unable to lock");
+ }
+
}
Status CyrusPerProcessData::Init()
@@ -405,6 +457,10 @@ Status CyrusPerProcessData::Init()
CyrusPerProcessData & CyrusPerProcessData::GetInstance()
{
+ // Meyer's singleton, thread safe and lazily initialized in C++11
+ //
+ // Must be lazily initialized to allow client code to plug in a GSSAPI mutex
+ // implementation.
static CyrusPerProcessData per_process_data;
return per_process_data;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22ea06a3/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/gsasl_engine.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/gsasl_engine.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/gsasl_engine.cc
index 8286bac..7705c81 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/gsasl_engine.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/gsasl_engine.cc
@@ -16,18 +16,26 @@
* limitations under the License.
*/
+#include "hdfspp/locks.h"
+
#include <sstream>
#include <gsasl.h>
#include "sasl_engine.h"
#include "gsasl_engine.h"
#include "common/logging.h"
+
namespace hdfs {
+
/*****************************************************************************
* GSASL UTILITY FUNCTIONS
*/
+static Mutex *getSaslMutex() {
+ return LockManager::getGssapiMutex();
+}
+
static Status rc_to_status(int rc)
{
if (rc == GSASL_OK) {
@@ -70,32 +78,45 @@ std::pair<Status, std::string> base64_encode(const std::string & in) {
GSaslEngine::~GSaslEngine()
{
- if (session_ != nullptr) {
+ // These should already be called in this->Finish
+ try {
+ LockGuard saslGuard(getSaslMutex());
+ if (session_ != nullptr) {
gsasl_finish(session_);
- }
+ }
- if (ctx_ != nullptr) {
+ if (ctx_ != nullptr) {
gsasl_done(ctx_);
+ }
+ } catch (const LockFailure& e) {
+ if(session_ || ctx_) {
+ LOG_ERROR(kRPC, << "GSaslEngine::~GSaslEngine@" << this << " unable to dispose of gsasl state: " << e.what());
+ }
}
}
Status GSaslEngine::gsasl_new() {
- int status = GSASL_OK;
-
- if (ctx_) return Status::OK();
-
- status = gsasl_init( & ctx_);
-
- switch ( status) {
- case GSASL_OK:
- return Status::OK();
- case GSASL_MALLOC_ERROR:
- LOG_WARN(kRPC, << "GSaslEngine: Out of memory.");
- return Status::Error("SaslEngine: Out of memory.");
- default:
- LOG_WARN(kRPC, << "GSaslEngine: Unexpected error." << status);
- return Status::Error("SaslEngine: Unexpected error.");
- }
+ int status = GSASL_OK;
+
+ if (ctx_) return Status::OK();
+
+ try {
+ LockGuard saslGuard(getSaslMutex());
+ status = gsasl_init( & ctx_);
+ } catch (const LockFailure& e) {
+ return Status::MutexError("Mutex that guards gsasl_init unable to lock");
+ }
+
+ switch ( status) {
+ case GSASL_OK:
+ return Status::OK();
+ case GSASL_MALLOC_ERROR:
+ LOG_WARN(kRPC, << "GSaslEngine: Out of memory.");
+ return Status::Error("SaslEngine: Out of memory.");
+ default:
+ LOG_WARN(kRPC, << "GSaslEngine: Unexpected error." << status);
+ return Status::Error("SaslEngine: Unexpected error.");
+ }
} // gsasl_new()
std::pair<Status, std::string>
@@ -107,12 +128,22 @@ GSaslEngine::Start()
this->gsasl_new();
/* Create new authentication session. */
- rc = gsasl_client_start(ctx_, chosen_mech_.mechanism.c_str(), &session_);
+ try {
+ LockGuard saslGuard(getSaslMutex());
+ rc = gsasl_client_start(ctx_, chosen_mech_.mechanism.c_str(), &session_);
+ } catch (const LockFailure& e) {
+ state_ = kErrorState;
+ return std::make_pair(Status::MutexError("Mutex that guards gsasl_client_start unable to lock"), "");
+ }
if (rc != GSASL_OK) {
state_ = kErrorState;
return std::make_pair( rc_to_status( rc), std::string(""));
}
- init_kerberos();
+ Status init_status = init_kerberos();
+ if(!init_status.ok()) {
+ state_ = kErrorState;
+ return std::make_pair(init_status, "");
+ }
state_ = kWaitingForData;
@@ -124,12 +155,17 @@ GSaslEngine::Start()
Status GSaslEngine::init_kerberos() {
//TODO: check that we have a principal
-
- gsasl_property_set(session_, GSASL_AUTHID, principal_.value().c_str());
- gsasl_property_set(session_, GSASL_HOSTNAME, chosen_mech_.serverid.c_str());
- gsasl_property_set(session_, GSASL_SERVICE, chosen_mech_.protocol.c_str());
- return Status::OK();
+ try {
+ LockGuard saslGuard(getSaslMutex());
+ // these don't return anything that indicates failure
+ gsasl_property_set(session_, GSASL_AUTHID, principal_.value().c_str());
+ gsasl_property_set(session_, GSASL_HOSTNAME, chosen_mech_.serverid.c_str());
+ gsasl_property_set(session_, GSASL_SERVICE, chosen_mech_.protocol.c_str());
+ } catch (const LockFailure& e) {
+ return Status::MutexError("Mutex that guards gsasl_property_set in GSaslEngine::init_kerberos unable to lock");
}
+ return Status::OK();
+}
std::pair<Status, std::string> GSaslEngine::Step(const std::string data) {
if (state_ != kWaitingForData)
@@ -137,8 +173,16 @@ std::pair<Status, std::string> GSaslEngine::Step(const std::string data) {
char * output = NULL;
size_t outputSize;
- int rc = gsasl_step(session_, data.c_str(), data.size(), &output,
+
+ int rc = 0;
+ try {
+ LockGuard saslGuard(getSaslMutex());
+ rc = gsasl_step(session_, data.c_str(), data.size(), &output,
&outputSize);
+ } catch (const LockFailure& e) {
+ state_ = kFailure;
+ return std::make_pair(Status::MutexError("Mutex that guards gsasl_client_start unable to lock"), "");
+ }
if (rc == GSASL_NEEDS_MORE || rc == GSASL_OK) {
std::string retval(output, output ? outputSize : 0);
@@ -166,16 +210,20 @@ Status GSaslEngine::Finish()
if (state_ != kSuccess && state_ != kFailure && state_ != kErrorState )
LOG_WARN(kRPC, << "GSaslEngine::finish when state is " << state_);
- if (session_ != nullptr) {
+ try {
+ LockGuard saslGuard(getSaslMutex());
+ if (session_ != nullptr) {
gsasl_finish(session_);
session_ = NULL;
- }
+ }
- if (ctx_ != nullptr) {
+ if (ctx_ != nullptr) {
gsasl_done(ctx_);
ctx_ = nullptr;
+ }
+ } catch (const LockFailure& e) {
+ return Status::MutexError("Mutex that guards sasl state cleanup in GSaslEngine::Finish unable to lock");
}
-
return Status::OK();
} // finish() method
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22ea06a3/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
index ad8191b..0957ea3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
@@ -38,8 +38,6 @@ namespace hdfs {
using namespace hadoop::common;
using namespace google::protobuf;
-template <class T>
-using optional = std::experimental::optional<T>;
/*****
* Threading model: all entry points need to acquire the sasl_lock before accessing
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22ea06a3/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
index 395fad5..0b4581e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
@@ -116,6 +116,9 @@ add_executable(hdfs_ioservice_test hdfs_ioservice_test.cc)
target_link_libraries(hdfs_ioservice_test fs gmock_main common ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
add_memcheck_test(hdfs_ioservice hdfs_ioservice_test)
+add_executable(user_lock_test user_lock_test.cc)
+target_link_libraries(user_lock_test fs gmock_main common ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
+add_memcheck_test(user_lock user_lock_test)
#
#
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22ea06a3/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/uri_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/uri_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/uri_test.cc
index 78f1a58..97f0afd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/uri_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/uri_test.cc
@@ -23,7 +23,6 @@ using ::testing::_;
using namespace hdfs;
-
URI expect_uri_throw(const char *uri) {
bool threw = false;
std::string what_msg;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22ea06a3/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/user_lock_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/user_lock_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/user_lock_test.cc
new file mode 100644
index 0000000..6df47b2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/user_lock_test.cc
@@ -0,0 +1,225 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <hdfspp/locks.h>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include <iostream>
+#include <memory>
+#include <mutex>
+#include <thread>
+#include <vector>
+
+using namespace hdfs;
+
+// try_lock will always return false, unlock will always throw because it
+// can never be locked.
+class CantLockMutex : public Mutex {
+ public:
+ void lock() override {
+ throw LockFailure("This mutex cannot be locked");
+ }
+ void unlock() override {
+ throw LockFailure("Unlock");
+ }
+ std::string str() override {
+ return "CantLockMutex";
+ }
+};
+
+TEST(UserLockTest, DefaultMutexBasics) {
+ Mutex *mtx = LockManager::TEST_get_default_mutex();
+
+ // lock and unlock twice to make sure unlock works
+ bool locked = false;
+ try {
+ mtx->lock();
+ locked = true;
+ } catch (...) {}
+ EXPECT_TRUE(locked);
+ mtx->unlock();
+
+ locked = false;
+ try {
+ mtx->lock();
+ locked = true;
+ } catch (...) {}
+ EXPECT_TRUE(locked);
+ mtx->unlock();
+
+ EXPECT_EQ(mtx->str(), "DefaultMutex");
+}
+
+
+// Make sure lock manager can only be initialized once unless test reset called
+TEST(UserLockTest, LockManager) {
+ std::unique_ptr<CantLockMutex> mtx(new CantLockMutex());
+ EXPECT_TRUE(mtx != nullptr);
+
+ // Check the default lock
+ Mutex *defaultGssapiMtx = LockManager::getGssapiMutex();
+ EXPECT_TRUE(defaultGssapiMtx != nullptr);
+
+ // Try a double init. Should not work
+ bool res = LockManager::InitLocks(mtx.get());
+ EXPECT_TRUE(res);
+
+ // Check pointer value
+ EXPECT_EQ(LockManager::getGssapiMutex(), mtx.get());
+
+ res = LockManager::InitLocks(mtx.get());
+ EXPECT_FALSE(res);
+
+ // Make sure test reset still works
+ LockManager::TEST_reset_manager();
+ res = LockManager::InitLocks(mtx.get());
+ EXPECT_TRUE(res);
+ LockManager::TEST_reset_manager();
+ EXPECT_EQ(LockManager::getGssapiMutex(), defaultGssapiMtx);
+}
+
+TEST(UserLockTest, CheckCantLockMutex) {
+ std::unique_ptr<CantLockMutex> mtx(new CantLockMutex());
+ EXPECT_TRUE(mtx != nullptr);
+
+ bool locked = false;
+ try {
+ mtx->lock();
+ } catch (...) {}
+ EXPECT_FALSE(locked);
+
+ bool threw_on_unlock = false;
+ try {
+ mtx->unlock();
+ } catch (const LockFailure& e) {
+ threw_on_unlock = true;
+ }
+ EXPECT_TRUE(threw_on_unlock);
+
+ EXPECT_EQ("CantLockMutex", mtx->str());
+}
+
+TEST(UserLockTest, LockGuardBasics) {
+ Mutex *goodMtx = LockManager::TEST_get_default_mutex();
+ CantLockMutex badMtx;
+
+ // lock/unlock a few times to increase chances of UB if lock is misused
+ for(int i=0;i<10;i++) {
+ bool caught_exception = false;
+ try {
+ LockGuard guard(goodMtx);
+ // now have a scoped lock
+ } catch (const LockFailure& e) {
+ caught_exception = true;
+ }
+ EXPECT_FALSE(caught_exception);
+ }
+
+ // still do a few times, but expect it to blow up each time
+ for(int i=0;i<10;i++) {
+ bool caught_exception = false;
+ try {
+ LockGuard guard(&badMtx);
+ // now have a scoped lock
+ } catch (const LockFailure& e) {
+ caught_exception = true;
+ }
+ EXPECT_TRUE(caught_exception);
+ }
+
+}
+
+struct Incrementer {
+ int64_t& _val;
+ int64_t _iters;
+ Mutex *_mtx;
+ Incrementer(int64_t &val, int64_t iters, Mutex *m)
+ : _val(val), _iters(iters), _mtx(m) {}
+ void operator()(){
+ for(int64_t i=0; i<_iters; i++) {
+ LockGuard valguard(_mtx);
+ _val += 1;
+ }
+ }
+};
+
+struct Decrementer {
+ int64_t& _val;
+ int64_t _iters;
+ Mutex *_mtx;
+ Decrementer(int64_t &val, int64_t iters, Mutex *m)
+ : _val(val), _iters(iters), _mtx(m) {}
+ void operator()(){
+ for(int64_t i=0; i<_iters; i++) {
+ LockGuard valguard(_mtx);
+ _val -= 1;
+ }
+ }
+};
+
+TEST(UserLockTest, LockGuardConcurrency) {
+ Mutex *mtx = LockManager::TEST_get_default_mutex();
+
+ // Prove that these actually mutate the value
+ int64_t test_value = 0;
+ Incrementer inc(test_value, 1000, mtx);
+ inc();
+ EXPECT_EQ(test_value, 1000);
+
+ Decrementer dec(test_value, 1000, mtx);
+ dec();
+ EXPECT_EQ(test_value, 0);
+
+ std::vector<std::thread> workers;
+ std::vector<Incrementer> incrementers;
+ std::vector<Decrementer> decrementors;
+
+ const int delta = 1024 * 1024;
+ const int threads = 2 * 6;
+ EXPECT_EQ(threads % 2, 0);
+
+ // a bunch of threads race to increment and decrement the value
+ // if all goes well the operations balance out and the value is unchanged
+ for(int i=0; i < threads; i++) {
+ if(i%2 == 0) {
+ incrementers.emplace_back(test_value, delta, mtx);
+ workers.emplace_back(incrementers.back());
+ } else {
+ decrementors.emplace_back(test_value, delta, mtx);
+ workers.emplace_back(decrementors.back());
+ }
+ }
+
+ // join, everything should balance to 0
+ for(std::thread& thread : workers) {
+ thread.join();
+ }
+ EXPECT_EQ(test_value, 0);
+}
+
+
+int main(int argc, char *argv[]) {
+
+ // The following line must be executed to initialize Google Mock
+ // (and Google Test) before running the tests.
+ ::testing::InitGoogleMock(&argc, argv);
+ int res = RUN_ALL_TESTS();
+ return res;
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/52] [abbrv] hadoop git commit: HDFS-11106: libhdfs++: Some
refactoring to better organize files. Contributed by James Clampffer.
Posted by jh...@apache.org.
HDFS-11106: libhdfs++: Some refactoring to better organize files. Contributed by James Clampffer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dd7837c4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dd7837c4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dd7837c4
Branch: refs/heads/HDFS-8707
Commit: dd7837c429f689e45f91248c1ee5b7d10e168fbf
Parents: 2524afb
Author: James <jh...@apache.org>
Authored: Tue Nov 29 18:09:53 2016 -0500
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../native/libhdfspp/lib/common/CMakeLists.txt | 2 +-
.../main/native/libhdfspp/lib/common/base64.cc | 73 ---
.../libhdfspp/lib/common/hdfs_ioservice.cc | 51 ++
.../libhdfspp/lib/common/hdfs_ioservice.h | 46 ++
.../libhdfspp/lib/common/hdfs_public_api.cc | 47 --
.../libhdfspp/lib/common/hdfs_public_api.h | 39 --
.../main/native/libhdfspp/lib/common/util.cc | 51 ++
.../lib/connection/datanodeconnection.h | 2 +-
.../main/native/libhdfspp/lib/fs/CMakeLists.txt | 2 +-
.../main/native/libhdfspp/lib/fs/filehandle.h | 2 +-
.../main/native/libhdfspp/lib/fs/filesystem.cc | 492 +---------------
.../native/libhdfspp/lib/fs/filesystem_sync.cc | 555 +++++++++++++++++++
.../main/native/libhdfspp/tests/CMakeLists.txt | 2 +-
.../libhdfspp/tests/sasl_digest_md5_test.cc | 3 +
14 files changed, 712 insertions(+), 655 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7837c4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
index 501d3b5..bdeb068 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
@@ -19,6 +19,6 @@ if(NEED_LINK_DL)
set(LIB_DL dl)
endif()
-add_library(common_obj OBJECT base64.cc status.cc sasl_digest_md5.cc hdfs_public_api.cc options.cc configuration.cc configuration_loader.cc hdfs_configuration.cc uri.cc util.cc retry_policy.cc cancel_tracker.cc logging.cc libhdfs_events_impl.cc auth_info.cc namenode_info.cc)
+add_library(common_obj OBJECT status.cc sasl_digest_md5.cc hdfs_ioservice.cc options.cc configuration.cc configuration_loader.cc hdfs_configuration.cc uri.cc util.cc retry_policy.cc cancel_tracker.cc logging.cc libhdfs_events_impl.cc auth_info.cc namenode_info.cc)
add_library(common $<TARGET_OBJECTS:common_obj> $<TARGET_OBJECTS:uriparser2_obj>)
target_link_libraries(common ${LIB_DL})
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7837c4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/base64.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/base64.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/base64.cc
deleted file mode 100644
index 76d10e6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/base64.cc
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "util.h"
-
-#include <array>
-#include <functional>
-#include <algorithm>
-
-namespace hdfs {
-
-std::string Base64Encode(const std::string &src) {
- //encoded size is (sizeof(buf) + 2) / 3 * 4
- static const std::string base64_chars =
- "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
- "abcdefghijklmnopqrstuvwxyz"
- "0123456789+/";
- std::string ret;
- int i = 0;
- int j = 0;
- unsigned char char_array_3[3];
- unsigned char char_array_4[4];
- unsigned const char *bytes_to_encode = reinterpret_cast<unsigned const char *>(&src[i]);
- unsigned int in_len = src.size();
-
- while (in_len--) {
- char_array_3[i++] = *(bytes_to_encode++);
- if (i == 3) {
- char_array_4[0] = (char_array_3[0] & 0xfc) >> 2;
- char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4);
- char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6);
- char_array_4[3] = char_array_3[2] & 0x3f;
-
- for(i = 0; (i <4) ; i++)
- ret += base64_chars[char_array_4[i]];
- i = 0;
- }
- }
-
- if (i) {
- for(j = i; j < 3; j++)
- char_array_3[j] = '\0';
-
- char_array_4[0] = (char_array_3[0] & 0xfc) >> 2;
- char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4);
- char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6);
- char_array_4[3] = char_array_3[2] & 0x3f;
-
- for (j = 0; (j < i + 1); j++)
- ret += base64_chars[char_array_4[j]];
-
- while((i++ < 3))
- ret += '=';
- }
- return ret;
-}
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7837c4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.cc
new file mode 100644
index 0000000..f6876cd
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.cc
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hdfs_ioservice.h"
+
+#include "common/logging.h"
+
+namespace hdfs {
+
+IoService::~IoService() {}
+
+IoService *IoService::New() { return new IoServiceImpl(); }
+
+void IoServiceImpl::Run() {
+ // The IoService executes callbacks provided by library users in the context of worker threads,
+ // there is no way of preventing those callbacks from throwing but we can at least prevent them
+ // from escaping this library and crashing the process.
+
+ // As recommended in http://www.boost.org/doc/libs/1_39_0/doc/html/boost_asio/reference/io_service.html#boost_asio.reference.io_service.effect_of_exceptions_thrown_from_handlers
+ asio::io_service::work work(io_service_);
+ for(;;)
+ {
+ try
+ {
+ io_service_.run();
+ break;
+ } catch (const std::exception & e) {
+ LOG_WARN(kFileSystem, << "Unexpected exception in libhdfspp worker thread: " << e.what());
+ } catch (...) {
+ LOG_WARN(kFileSystem, << "Unexpected value not derived from std::exception in libhdfspp worker thread");
+ }
+ }
+}
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7837c4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.h
new file mode 100644
index 0000000..73d167e
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.h
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef COMMON_HDFS_IOSERVICE_H_
+#define COMMON_HDFS_IOSERVICE_H_
+
+#include "hdfspp/hdfspp.h"
+
+#include <asio/io_service.hpp>
+
+namespace hdfs {
+
+/*
+ * A thin wrapper over the asio::io_service.
+ * -In the future this could own the worker threads that execute io tasks which
+ * makes it easier to share IoServices between FileSystems. See HDFS-10796 for
+ * rationale.
+ */
+
+class IoServiceImpl : public IoService {
+ public:
+ virtual void Run() override;
+ virtual void Stop() override { io_service_.stop(); }
+ ::asio::io_service &io_service() { return io_service_; }
+ private:
+ ::asio::io_service io_service_;
+};
+
+}
+
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7837c4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_public_api.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_public_api.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_public_api.cc
deleted file mode 100644
index 188071d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_public_api.cc
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "hdfs_public_api.h"
-
-#include "common/logging.h"
-
-namespace hdfs {
-
-IoService::~IoService() {}
-
-IoService *IoService::New() { return new IoServiceImpl(); }
-
-void IoServiceImpl::Run() {
- // As recommended in http://www.boost.org/doc/libs/1_39_0/doc/html/boost_asio/reference/io_service.html#boost_asio.reference.io_service.effect_of_exceptions_thrown_from_handlers
- asio::io_service::work work(io_service_);
- for(;;)
- {
- try
- {
- io_service_.run();
- break;
- } catch (const std::exception & e) {
- LOG_WARN(kFileSystem, << "Unexpected exception in libhdfspp worker thread: " << e.what());
- } catch (...) {
- LOG_WARN(kFileSystem, << "Unexpected value not derived from std::exception in libhdfspp worker thread");
- }
- }
-}
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7837c4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_public_api.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_public_api.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_public_api.h
deleted file mode 100644
index c8fea5e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_public_api.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef COMMON_HDFS_PUBLIC_API_H_
-#define COMMON_HDFS_PUBLIC_API_H_
-
-#include "hdfspp/hdfspp.h"
-
-#include <asio/io_service.hpp>
-
-namespace hdfs {
-
-class IoServiceImpl : public IoService {
- public:
- virtual void Run() override;
- virtual void Stop() override { io_service_.stop(); }
- ::asio::io_service &io_service() { return io_service_; }
- private:
- ::asio::io_service io_service_;
-};
-
-}
-
-#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7837c4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
index ede6acd..375f951 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
@@ -20,12 +20,14 @@
#include "common/util_c.h"
#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
+
#include <exception>
#include <sstream>
#include <iostream>
#include <iomanip>
#include <thread>
+
namespace hdfs {
bool ReadDelimitedPBMessage(::google::protobuf::io::CodedInputStream *in,
@@ -73,6 +75,53 @@ std::string GetRandomClientName() {
return oss.str();
}
+std::string Base64Encode(const std::string &src) {
+ //encoded size is (sizeof(buf) + 2) / 3 * 4
+ static const std::string base64_chars =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ "abcdefghijklmnopqrstuvwxyz"
+ "0123456789+/";
+ std::string ret;
+ int i = 0;
+ int j = 0;
+ unsigned char char_array_3[3];
+ unsigned char char_array_4[4];
+ unsigned const char *bytes_to_encode = reinterpret_cast<unsigned const char *>(&src[i]);
+ unsigned int in_len = src.size();
+
+ while (in_len--) {
+ char_array_3[i++] = *(bytes_to_encode++);
+ if (i == 3) {
+ char_array_4[0] = (char_array_3[0] & 0xfc) >> 2;
+ char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4);
+ char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6);
+ char_array_4[3] = char_array_3[2] & 0x3f;
+
+ for(i = 0; (i <4) ; i++)
+ ret += base64_chars[char_array_4[i]];
+ i = 0;
+ }
+ }
+
+ if (i) {
+ for(j = i; j < 3; j++)
+ char_array_3[j] = '\0';
+
+ char_array_4[0] = (char_array_3[0] & 0xfc) >> 2;
+ char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4);
+ char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6);
+ char_array_4[3] = char_array_3[2] & 0x3f;
+
+ for (j = 0; (j < i + 1); j++)
+ ret += base64_chars[char_array_4[j]];
+
+ while((i++ < 3))
+ ret += '=';
+ }
+ return ret;
+}
+
+
std::string SafeDisconnect(asio::ip::tcp::socket *sock) {
std::string err;
if(sock && sock->is_open()) {
@@ -117,3 +166,5 @@ bool IsHighBitSet(uint64_t num) {
void ShutdownProtobufLibrary_C() {
google::protobuf::ShutdownProtobufLibrary();
}
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7837c4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/datanodeconnection.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/datanodeconnection.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/datanodeconnection.h
index 0ff6fc4..21193b3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/datanodeconnection.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/datanodeconnection.h
@@ -18,7 +18,7 @@
#ifndef LIBHDFSPP_LIB_CONNECTION_DATANODECONNECTION_H_
#define LIBHDFSPP_LIB_CONNECTION_DATANODECONNECTION_H_
-#include "common/hdfs_public_api.h"
+#include "common/hdfs_ioservice.h"
#include "common/async_stream.h"
#include "ClientNamenodeProtocol.pb.h"
#include "common/libhdfs_events_impl.h"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7837c4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/CMakeLists.txt
index 0bce70d..624cda5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/CMakeLists.txt
@@ -16,6 +16,6 @@
# limitations under the License.
#
-add_library(fs_obj OBJECT filesystem.cc filehandle.cc bad_datanode_tracker.cc namenode_operations.cc)
+add_library(fs_obj OBJECT filesystem.cc filesystem_sync.cc filehandle.cc bad_datanode_tracker.cc namenode_operations.cc)
add_dependencies(fs_obj proto)
add_library(fs $<TARGET_OBJECTS:fs_obj>)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7837c4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
index 7e7c79d..14f7ea8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
@@ -18,7 +18,7 @@
#ifndef LIBHDFSPP_LIB_FS_FILEHANDLE_H_
#define LIBHDFSPP_LIB_FS_FILEHANDLE_H_
-#include "common/hdfs_public_api.h"
+#include "common/hdfs_ioservice.h"
#include "common/async_stream.h"
#include "common/cancel_tracker.h"
#include "common/libhdfs_events_impl.h"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7837c4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
index 5d5b9f2..d805716 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
@@ -32,8 +32,7 @@
namespace hdfs {
-static const char kNamenodeProtocol[] =
- "org.apache.hadoop.hdfs.protocol.ClientProtocol";
+static const char kNamenodeProtocol[] = "org.apache.hadoop.hdfs.protocol.ClientProtocol";
static const int kNamenodeProtocolVersion = 1;
using ::asio::ip::tcp;
@@ -203,26 +202,6 @@ void FileSystemImpl::Connect(const std::string &server,
});
}
-Status FileSystemImpl::Connect(const std::string &server, const std::string &service) {
- LOG_INFO(kFileSystem, << "FileSystemImpl::[sync]Connect(" << FMT_THIS_ADDR
- << ", server=" << server << ", service=" << service << ") called");
-
- /* synchronized */
- auto stat = std::make_shared<std::promise<Status>>();
- std::future<Status> future = stat->get_future();
-
- auto callback = [stat](const Status &s, FileSystem *fs) {
- (void)fs;
- stat->set_value(s);
- };
-
- Connect(server, service, callback);
-
- /* block until promise is set */
- auto s = future.get();
-
- return s;
-}
void FileSystemImpl::ConnectToDefaultFs(const std::function<void(const Status &, FileSystem *)> &handler) {
std::string scheme = options_.defaultFS.get_scheme();
@@ -248,25 +227,6 @@ void FileSystemImpl::ConnectToDefaultFs(const std::function<void(const Status &,
Connect(host, port_as_string, handler);
}
-Status FileSystemImpl::ConnectToDefaultFs() {
- auto stat = std::make_shared<std::promise<Status>>();
- std::future<Status> future = stat->get_future();
-
- auto callback = [stat](const Status &s, FileSystem *fs) {
- (void)fs;
- stat->set_value(s);
- };
-
- ConnectToDefaultFs(callback);
-
- /* block until promise is set */
- auto s = future.get();
-
- return s;
-}
-
-
-
int FileSystemImpl::AddWorkerThread() {
LOG_DEBUG(kFileSystem, << "FileSystemImpl::AddWorkerThread("
<< FMT_THIS_ADDR << ") called."
@@ -297,38 +257,6 @@ void FileSystemImpl::Open(
});
}
-Status FileSystemImpl::Open(const std::string &path,
- FileHandle **handle) {
- LOG_DEBUG(kFileSystem, << "FileSystemImpl::[sync]Open("
- << FMT_THIS_ADDR << ", path="
- << path << ") called");
-
- auto callstate = std::make_shared<std::promise<std::tuple<Status, FileHandle*>>>();
- std::future<std::tuple<Status, FileHandle*>> future(callstate->get_future());
-
- /* wrap async FileSystem::Open with promise to make it a blocking call */
- auto h = [callstate](const Status &s, FileHandle *is) {
- callstate->set_value(std::make_tuple(s, is));
- };
-
- Open(path, h);
-
- /* block until promise is set */
- auto returnstate = future.get();
- Status stat = std::get<0>(returnstate);
- FileHandle *file_handle = std::get<1>(returnstate);
-
- if (!stat.ok()) {
- delete file_handle;
- return stat;
- }
- if (!file_handle) {
- return stat;
- }
-
- *handle = file_handle;
- return stat;
-}
BlockLocation LocatedBlockToBlockLocation(const hadoop::hdfs::LocatedBlockProto & locatedBlock)
{
@@ -411,39 +339,6 @@ void FileSystemImpl::GetBlockLocations(const std::string & path, uint64_t offset
nn_.GetBlockLocations(path, offset, length, conversion);
}
-Status FileSystemImpl::GetBlockLocations(const std::string & path, uint64_t offset, uint64_t length,
- std::shared_ptr<FileBlockLocation> * fileBlockLocations)
-{
- LOG_DEBUG(kFileSystem, << "FileSystemImpl::[sync]GetBlockLocations("
- << FMT_THIS_ADDR << ", path="
- << path << ") called");
-
- if (!fileBlockLocations)
- return Status::InvalidArgument("Null pointer passed to GetBlockLocations");
-
- auto callstate = std::make_shared<std::promise<std::tuple<Status, std::shared_ptr<FileBlockLocation>>>>();
- std::future<std::tuple<Status, std::shared_ptr<FileBlockLocation>>> future(callstate->get_future());
-
- /* wrap async call with promise/future to make it blocking */
- auto callback = [callstate](const Status &s, std::shared_ptr<FileBlockLocation> blockInfo) {
- callstate->set_value(std::make_tuple(s,blockInfo));
- };
-
- GetBlockLocations(path, offset, length, callback);
-
- /* wait for async to finish */
- auto returnstate = future.get();
- auto stat = std::get<0>(returnstate);
-
- if (!stat.ok()) {
- return stat;
- }
-
- *fileBlockLocations = std::get<1>(returnstate);
-
- return stat;
-}
-
void FileSystemImpl::GetPreferredBlockSize(const std::string &path,
const std::function<void(const Status &, const uint64_t &)> &handler) {
LOG_DEBUG(kFileSystem, << "FileSystemImpl::GetPreferredBlockSize("
@@ -453,33 +348,6 @@ void FileSystemImpl::GetPreferredBlockSize(const std::string &path,
nn_.GetPreferredBlockSize(path, handler);
}
-Status FileSystemImpl::GetPreferredBlockSize(const std::string &path, uint64_t & block_size) {
- LOG_DEBUG(kFileSystem, << "FileSystemImpl::[sync]GetPreferredBlockSize("
- << FMT_THIS_ADDR << ", path="
- << path << ") called");
-
- auto callstate = std::make_shared<std::promise<std::tuple<Status, uint64_t>>>();
- std::future<std::tuple<Status, uint64_t>> future(callstate->get_future());
-
- /* wrap async FileSystem::GetPreferredBlockSize with promise to make it a blocking call */
- auto h = [callstate](const Status &s, const uint64_t & bsize) {
- callstate->set_value(std::make_tuple(s, bsize));
- };
-
- GetPreferredBlockSize(path, h);
-
- /* block until promise is set */
- auto returnstate = future.get();
- Status stat = std::get<0>(returnstate);
- uint64_t size = std::get<1>(returnstate);
-
- if (!stat.ok()) {
- return stat;
- }
-
- block_size = size;
- return stat;
-}
void FileSystemImpl::SetReplication(const std::string & path, int16_t replication, std::function<void(const Status &)> handler) {
LOG_DEBUG(kFileSystem,
@@ -499,27 +367,6 @@ void FileSystemImpl::SetReplication(const std::string & path, int16_t replicatio
nn_.SetReplication(path, replication, handler);
}
-Status FileSystemImpl::SetReplication(const std::string & path, int16_t replication) {
- LOG_DEBUG(kFileSystem,
- << "FileSystemImpl::[sync]SetReplication(" << FMT_THIS_ADDR << ", path=" << path <<
- ", replication=" << replication << ") called");
-
- auto callstate = std::make_shared<std::promise<Status>>();
- std::future<Status> future(callstate->get_future());
-
- /* wrap async FileSystem::SetReplication with promise to make it a blocking call */
- auto h = [callstate](const Status &s) {
- callstate->set_value(s);
- };
-
- SetReplication(path, replication, h);
-
- /* block until promise is set */
- auto returnstate = future.get();
- Status stat = returnstate;
-
- return stat;
-}
void FileSystemImpl::SetTimes(const std::string & path, uint64_t mtime, uint64_t atime,
std::function<void(const Status &)> handler) {
@@ -535,27 +382,6 @@ void FileSystemImpl::SetTimes(const std::string & path, uint64_t mtime, uint64_t
nn_.SetTimes(path, mtime, atime, handler);
}
-Status FileSystemImpl::SetTimes(const std::string & path, uint64_t mtime, uint64_t atime) {
- LOG_DEBUG(kFileSystem,
- << "FileSystemImpl::[sync]SetTimes(" << FMT_THIS_ADDR << ", path=" << path <<
- ", mtime=" << mtime << ", atime=" << atime << ") called");
-
- auto callstate = std::make_shared<std::promise<Status>>();
- std::future<Status> future(callstate->get_future());
-
- /* wrap async FileSystem::SetTimes with promise to make it a blocking call */
- auto h = [callstate](const Status &s) {
- callstate->set_value(s);
- };
-
- SetTimes(path, mtime, atime, h);
-
- /* block until promise is set */
- auto returnstate = future.get();
- Status stat = returnstate;
-
- return stat;
-}
void FileSystemImpl::GetFileInfo(
const std::string &path,
@@ -567,34 +393,6 @@ void FileSystemImpl::GetFileInfo(
nn_.GetFileInfo(path, handler);
}
-Status FileSystemImpl::GetFileInfo(const std::string &path,
- StatInfo & stat_info) {
- LOG_DEBUG(kFileSystem, << "FileSystemImpl::[sync]GetFileInfo("
- << FMT_THIS_ADDR << ", path="
- << path << ") called");
-
- auto callstate = std::make_shared<std::promise<std::tuple<Status, StatInfo>>>();
- std::future<std::tuple<Status, StatInfo>> future(callstate->get_future());
-
- /* wrap async FileSystem::GetFileInfo with promise to make it a blocking call */
- auto h = [callstate](const Status &s, const StatInfo &si) {
- callstate->set_value(std::make_tuple(s, si));
- };
-
- GetFileInfo(path, h);
-
- /* block until promise is set */
- auto returnstate = future.get();
- Status stat = std::get<0>(returnstate);
- StatInfo info = std::get<1>(returnstate);
-
- if (!stat.ok()) {
- return stat;
- }
-
- stat_info = info;
- return stat;
-}
void FileSystemImpl::GetFsStats(
const std::function<void(const Status &, const FsInfo &)> &handler) {
@@ -604,32 +402,6 @@ void FileSystemImpl::GetFsStats(
nn_.GetFsStats(handler);
}
-Status FileSystemImpl::GetFsStats(FsInfo & fs_info) {
- LOG_DEBUG(kFileSystem,
- << "FileSystemImpl::[sync]GetFsStats(" << FMT_THIS_ADDR << ") called");
-
- auto callstate = std::make_shared<std::promise<std::tuple<Status, FsInfo>>>();
- std::future<std::tuple<Status, FsInfo>> future(callstate->get_future());
-
- /* wrap async FileSystem::GetFsStats with promise to make it a blocking call */
- auto h = [callstate](const Status &s, const FsInfo &si) {
- callstate->set_value(std::make_tuple(s, si));
- };
-
- GetFsStats(h);
-
- /* block until promise is set */
- auto returnstate = future.get();
- Status stat = std::get<0>(returnstate);
- FsInfo info = std::get<1>(returnstate);
-
- if (!stat.ok()) {
- return stat;
- }
-
- fs_info = info;
- return stat;
-}
/**
* Helper function for recursive GetListing calls.
@@ -666,43 +438,6 @@ void FileSystemImpl::GetListing(
nn_.GetListing(path, callback);
}
-Status FileSystemImpl::GetListing(const std::string &path, std::vector<StatInfo> * stat_infos) {
- LOG_DEBUG(kFileSystem, << "FileSystemImpl::[sync]GetListing("
- << FMT_THIS_ADDR << ", path="
- << path << ") called");
-
- if (!stat_infos) {
- return Status::InvalidArgument("FileSystemImpl::GetListing: argument 'stat_infos' cannot be NULL");
- }
-
- auto callstate = std::make_shared<std::promise<Status>>();
- std::future<Status> future(callstate->get_future());
-
- /* wrap async FileSystem::GetListing with promise to make it a blocking call.
- *
- Keep requesting more until we get the entire listing, and don't set the promise
- * until we have the entire listing.
- */
- auto h = [callstate, stat_infos](const Status &s, const std::vector<StatInfo> & si, bool has_more) -> bool {
- if (!si.empty()) {
- stat_infos->insert(stat_infos->end(), si.begin(), si.end());
- }
-
- bool done = !s.ok() || !has_more;
- if (done) {
- callstate->set_value(s);
- return false;
- }
- return true;
- };
-
- GetListing(path, h);
-
- /* block until promise is set */
- Status stat = future.get();
-
- return stat;
-}
void FileSystemImpl::Mkdirs(const std::string & path, uint16_t permissions, bool createparent,
std::function<void(const Status &)> handler) {
@@ -724,27 +459,6 @@ void FileSystemImpl::Mkdirs(const std::string & path, uint16_t permissions, bool
nn_.Mkdirs(path, permissions, createparent, handler);
}
-Status FileSystemImpl::Mkdirs(const std::string & path, uint16_t permissions, bool createparent) {
- LOG_DEBUG(kFileSystem,
- << "FileSystemImpl::[sync]Mkdirs(" << FMT_THIS_ADDR << ", path=" << path <<
- ", permissions=" << permissions << ", createparent=" << createparent << ") called");
-
- auto callstate = std::make_shared<std::promise<Status>>();
- std::future<Status> future(callstate->get_future());
-
- /* wrap async FileSystem::Mkdirs with promise to make it a blocking call */
- auto h = [callstate](const Status &s) {
- callstate->set_value(s);
- };
-
- Mkdirs(path, permissions, createparent, h);
-
- /* block until promise is set */
- auto returnstate = future.get();
- Status stat = returnstate;
-
- return stat;
-}
void FileSystemImpl::Delete(const std::string &path, bool recursive,
const std::function<void(const Status &)> &handler) {
@@ -759,26 +473,6 @@ void FileSystemImpl::Delete(const std::string &path, bool recursive,
nn_.Delete(path, recursive, handler);
}
-Status FileSystemImpl::Delete(const std::string &path, bool recursive) {
- LOG_DEBUG(kFileSystem,
- << "FileSystemImpl::[sync]Delete(" << FMT_THIS_ADDR << ", path=" << path << ", recursive=" << recursive << ") called");
-
- auto callstate = std::make_shared<std::promise<Status>>();
- std::future<Status> future(callstate->get_future());
-
- /* wrap async FileSystem::Delete with promise to make it a blocking call */
- auto h = [callstate](const Status &s) {
- callstate->set_value(s);
- };
-
- Delete(path, recursive, h);
-
- /* block until promise is set */
- auto returnstate = future.get();
- Status stat = returnstate;
-
- return stat;
-}
void FileSystemImpl::Rename(const std::string &oldPath, const std::string &newPath,
const std::function<void(const Status &)> &handler) {
@@ -798,26 +492,6 @@ void FileSystemImpl::Rename(const std::string &oldPath, const std::string &newPa
nn_.Rename(oldPath, newPath, handler);
}
-Status FileSystemImpl::Rename(const std::string &oldPath, const std::string &newPath) {
- LOG_DEBUG(kFileSystem,
- << "FileSystemImpl::[sync]Rename(" << FMT_THIS_ADDR << ", oldPath=" << oldPath << ", newPath=" << newPath << ") called");
-
- auto callstate = std::make_shared<std::promise<Status>>();
- std::future<Status> future(callstate->get_future());
-
- /* wrap async FileSystem::Rename with promise to make it a blocking call */
- auto h = [callstate](const Status &s) {
- callstate->set_value(s);
- };
-
- Rename(oldPath, newPath, h);
-
- /* block until promise is set */
- auto returnstate = future.get();
- Status stat = returnstate;
-
- return stat;
-}
void FileSystemImpl::SetPermission(const std::string & path,
uint16_t permissions, const std::function<void(const Status &)> &handler) {
@@ -837,25 +511,6 @@ void FileSystemImpl::SetPermission(const std::string & path,
nn_.SetPermission(path, permissions, handler);
}
-Status FileSystemImpl::SetPermission(const std::string & path, uint16_t permissions) {
- LOG_DEBUG(kFileSystem,
- << "FileSystemImpl::[sync]SetPermission(" << FMT_THIS_ADDR << ", path=" << path << ", permissions=" << permissions << ") called");
-
- auto callstate = std::make_shared<std::promise<Status>>();
- std::future<Status> future(callstate->get_future());
-
- /* wrap async FileSystem::SetPermission with promise to make it a blocking call */
- auto h = [callstate](const Status &s) {
- callstate->set_value(s);
- };
-
- SetPermission(path, permissions, h);
-
- /* block until promise is set */
- Status stat = future.get();
-
- return stat;
-}
void FileSystemImpl::SetOwner(const std::string & path, const std::string & username,
const std::string & groupname, const std::function<void(const Status &)> &handler) {
@@ -870,25 +525,6 @@ void FileSystemImpl::SetOwner(const std::string & path, const std::string & user
nn_.SetOwner(path, username, groupname, handler);
}
-Status FileSystemImpl::SetOwner(const std::string & path, const std::string & username,
- const std::string & groupname) {
- LOG_DEBUG(kFileSystem,
- << "FileSystemImpl::[sync]SetOwner(" << FMT_THIS_ADDR << ", path=" << path << ", username=" << username << ", groupname=" << groupname << ") called");
-
- auto callstate = std::make_shared<std::promise<Status>>();
- std::future<Status> future(callstate->get_future());
-
- /* wrap async FileSystem::SetOwner with promise to make it a blocking call */
- auto h = [callstate](const Status &s) {
- callstate->set_value(s);
- };
-
- SetOwner(path, username, groupname, h);
-
- /* block until promise is set */
- Status stat = future.get();
- return stat;
-}
/**
* Helper function for recursive Find calls.
@@ -1016,50 +652,6 @@ void FileSystemImpl::Find(
nn_.GetListing("/", callback);
}
-Status FileSystemImpl::Find(const std::string &path, const std::string &name, const uint32_t maxdepth, std::vector<StatInfo> * stat_infos) {
- LOG_DEBUG(kFileSystem, << "FileSystemImpl::[sync]Find("
- << FMT_THIS_ADDR << ", path="
- << path << ", name="
- << name << ") called");
-
- if (!stat_infos) {
- return Status::InvalidArgument("FileSystemImpl::Find: argument 'stat_infos' cannot be NULL");
- }
-
- // In this case, we're going to have the async code populate stat_infos.
-
- std::promise<void> promise = std::promise<void>();
- std::future<void> future(promise.get_future());
- Status status = Status::OK();
-
- /**
- * Keep requesting more until we get the entire listing. Set the promise
- * when we have the entire listing to stop.
- *
- * Find guarantees that the handler will only be called once at a time,
- * so we do not need any locking here
- */
- auto h = [&status, &promise, stat_infos](const Status &s, const std::vector<StatInfo> & si, bool has_more_results) -> bool {
- if (!si.empty()) {
- stat_infos->insert(stat_infos->end(), si.begin(), si.end());
- }
- if (!s.ok() && status.ok()){
- //We make sure we set 'status' only on the first error.
- status = s;
- }
- if (!has_more_results) {
- promise.set_value();
- return false;
- }
- return true;
- };
-
- Find(path, name, maxdepth, h);
-
- /* block until promise is set */
- future.get();
- return status;
-}
void FileSystemImpl::CreateSnapshot(const std::string &path,
const std::string &name,
@@ -1075,27 +667,6 @@ void FileSystemImpl::CreateSnapshot(const std::string &path,
nn_.CreateSnapshot(path, name, handler);
}
-Status FileSystemImpl::CreateSnapshot(const std::string &path,
- const std::string &name) {
- LOG_DEBUG(kFileSystem,
- << "FileSystemImpl::[sync]CreateSnapshot(" << FMT_THIS_ADDR << ", path=" << path << ", name=" << name << ") called");
-
- auto callstate = std::make_shared<std::promise<Status>>();
- std::future<Status> future(callstate->get_future());
-
- /* wrap async FileSystem::CreateSnapshot with promise to make it a blocking call */
- auto h = [callstate](const Status &s) {
- callstate->set_value(s);
- };
-
- CreateSnapshot(path, name, h);
-
- /* block until promise is set */
- auto returnstate = future.get();
- Status stat = returnstate;
-
- return stat;
-}
void FileSystemImpl::DeleteSnapshot(const std::string &path,
const std::string &name,
@@ -1115,27 +686,6 @@ void FileSystemImpl::DeleteSnapshot(const std::string &path,
nn_.DeleteSnapshot(path, name, handler);
}
-Status FileSystemImpl::DeleteSnapshot(const std::string &path,
- const std::string &name) {
- LOG_DEBUG(kFileSystem,
- << "FileSystemImpl::[sync]DeleteSnapshot(" << FMT_THIS_ADDR << ", path=" << path << ", name=" << name << ") called");
-
- auto callstate = std::make_shared<std::promise<Status>>();
- std::future<Status> future(callstate->get_future());
-
- /* wrap async FileSystem::DeleteSnapshot with promise to make it a blocking call */
- auto h = [callstate](const Status &s) {
- callstate->set_value(s);
- };
-
- DeleteSnapshot(path, name, h);
-
- /* block until promise is set */
- auto returnstate = future.get();
- Status stat = returnstate;
-
- return stat;
-}
void FileSystemImpl::AllowSnapshot(const std::string &path,
const std::function<void(const Status &)> &handler) {
@@ -1150,26 +700,6 @@ void FileSystemImpl::AllowSnapshot(const std::string &path,
nn_.AllowSnapshot(path, handler);
}
-Status FileSystemImpl::AllowSnapshot(const std::string &path) {
- LOG_DEBUG(kFileSystem,
- << "FileSystemImpl::[sync]AllowSnapshot(" << FMT_THIS_ADDR << ", path=" << path << ") called");
-
- auto callstate = std::make_shared<std::promise<Status>>();
- std::future<Status> future(callstate->get_future());
-
- /* wrap async FileSystem::AllowSnapshot with promise to make it a blocking call */
- auto h = [callstate](const Status &s) {
- callstate->set_value(s);
- };
-
- AllowSnapshot(path, h);
-
- /* block until promise is set */
- auto returnstate = future.get();
- Status stat = returnstate;
-
- return stat;
-}
void FileSystemImpl::DisallowSnapshot(const std::string &path,
const std::function<void(const Status &)> &handler) {
@@ -1184,26 +714,6 @@ void FileSystemImpl::DisallowSnapshot(const std::string &path,
nn_.DisallowSnapshot(path, handler);
}
-Status FileSystemImpl::DisallowSnapshot(const std::string &path) {
- LOG_DEBUG(kFileSystem,
- << "FileSystemImpl::[sync]DisallowSnapshot(" << FMT_THIS_ADDR << ", path=" << path << ") called");
-
- auto callstate = std::make_shared<std::promise<Status>>();
- std::future<Status> future(callstate->get_future());
-
- /* wrap async FileSystem::DisallowSnapshot with promise to make it a blocking call */
- auto h = [callstate](const Status &s) {
- callstate->set_value(s);
- };
-
- DisallowSnapshot(path, h);
-
- /* block until promise is set */
- auto returnstate = future.get();
- Status stat = returnstate;
-
- return stat;
-}
void FileSystemImpl::WorkerDeleter::operator()(std::thread *t) {
// It is far too easy to destroy the filesystem (and thus the threadpool)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7837c4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem_sync.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem_sync.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem_sync.cc
new file mode 100644
index 0000000..73be538
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem_sync.cc
@@ -0,0 +1,555 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "filesystem.h"
+
+#include <future>
+#include <tuple>
+
+#define FMT_THIS_ADDR "this=" << (void*)this
+
+// Note: This is just a place to hold boilerplate async to sync shim code,
+// place actual filesystem logic in filesystem.cc
+//
+//
+// Shim pattern pseudocode
+//
+// Status MySynchronizedMethod(method_args):
+// let stat = a promise<Status> wrapped in a shared_ptr
+//
+// Create a lambda that captures stat and any other variables that need to
+// be set based on the async operation. When invoked set variables with the
+// arguments passed (possibly do some translation), then set stat to indicate
+// the return status of the async call.
+//
+// invoke MyAsyncMethod(method_args, handler_lambda)
+//
+// block until stat value has been set while async work takes place
+//
+// return stat
+
+namespace hdfs {
+
+Status FileSystemImpl::Connect(const std::string &server, const std::string &service) {
+ LOG_INFO(kFileSystem, << "FileSystemImpl::[sync]Connect(" << FMT_THIS_ADDR
+ << ", server=" << server << ", service=" << service << ") called");
+
+ /* synchronized */
+ auto stat = std::make_shared<std::promise<Status>>();
+ std::future<Status> future = stat->get_future();
+
+ auto callback = [stat](const Status &s, FileSystem *fs) {
+ (void)fs;
+ stat->set_value(s);
+ };
+
+ Connect(server, service, callback);
+
+ /* block until promise is set */
+ auto s = future.get();
+
+ return s;
+}
+
+
+Status FileSystemImpl::ConnectToDefaultFs() {
+ auto stat = std::make_shared<std::promise<Status>>();
+ std::future<Status> future = stat->get_future();
+
+ auto callback = [stat](const Status &s, FileSystem *fs) {
+ (void)fs;
+ stat->set_value(s);
+ };
+
+ ConnectToDefaultFs(callback);
+
+ /* block until promise is set */
+ auto s = future.get();
+
+ return s;
+}
+
+
+Status FileSystemImpl::Open(const std::string &path,
+ FileHandle **handle) {
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl::[sync]Open("
+ << FMT_THIS_ADDR << ", path="
+ << path << ") called");
+
+ auto callstate = std::make_shared<std::promise<std::tuple<Status, FileHandle*>>>();
+ std::future<std::tuple<Status, FileHandle*>> future(callstate->get_future());
+
+ /* wrap async FileSystem::Open with promise to make it a blocking call */
+ auto h = [callstate](const Status &s, FileHandle *is) {
+ callstate->set_value(std::make_tuple(s, is));
+ };
+
+ Open(path, h);
+
+ /* block until promise is set */
+ auto returnstate = future.get();
+ Status stat = std::get<0>(returnstate);
+ FileHandle *file_handle = std::get<1>(returnstate);
+
+ if (!stat.ok()) {
+ delete file_handle;
+ return stat;
+ }
+ if (!file_handle) {
+ return stat;
+ }
+
+ *handle = file_handle;
+ return stat;
+}
+
+Status FileSystemImpl::GetBlockLocations(const std::string & path, uint64_t offset, uint64_t length,
+ std::shared_ptr<FileBlockLocation> * fileBlockLocations)
+{
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl::[sync]GetBlockLocations("
+ << FMT_THIS_ADDR << ", path="
+ << path << ") called");
+
+ if (!fileBlockLocations)
+ return Status::InvalidArgument("Null pointer passed to GetBlockLocations");
+
+ auto callstate = std::make_shared<std::promise<std::tuple<Status, std::shared_ptr<FileBlockLocation>>>>();
+ std::future<std::tuple<Status, std::shared_ptr<FileBlockLocation>>> future(callstate->get_future());
+
+ /* wrap async call with promise/future to make it blocking */
+ auto callback = [callstate](const Status &s, std::shared_ptr<FileBlockLocation> blockInfo) {
+ callstate->set_value(std::make_tuple(s,blockInfo));
+ };
+
+ GetBlockLocations(path, offset, length, callback);
+
+ /* wait for async to finish */
+ auto returnstate = future.get();
+ auto stat = std::get<0>(returnstate);
+
+ if (!stat.ok()) {
+ return stat;
+ }
+
+ *fileBlockLocations = std::get<1>(returnstate);
+
+ return stat;
+}
+
+Status FileSystemImpl::GetPreferredBlockSize(const std::string &path, uint64_t & block_size) {
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl::[sync]GetPreferredBlockSize("
+ << FMT_THIS_ADDR << ", path="
+ << path << ") called");
+
+ auto callstate = std::make_shared<std::promise<std::tuple<Status, uint64_t>>>();
+ std::future<std::tuple<Status, uint64_t>> future(callstate->get_future());
+
+ /* wrap async FileSystem::GetPreferredBlockSize with promise to make it a blocking call */
+ auto h = [callstate](const Status &s, const uint64_t & bsize) {
+ callstate->set_value(std::make_tuple(s, bsize));
+ };
+
+ GetPreferredBlockSize(path, h);
+
+ /* block until promise is set */
+ auto returnstate = future.get();
+ Status stat = std::get<0>(returnstate);
+ uint64_t size = std::get<1>(returnstate);
+
+ if (!stat.ok()) {
+ return stat;
+ }
+
+ block_size = size;
+ return stat;
+}
+
+Status FileSystemImpl::SetReplication(const std::string & path, int16_t replication) {
+ LOG_DEBUG(kFileSystem,
+ << "FileSystemImpl::[sync]SetReplication(" << FMT_THIS_ADDR << ", path=" << path <<
+ ", replication=" << replication << ") called");
+
+ auto callstate = std::make_shared<std::promise<Status>>();
+ std::future<Status> future(callstate->get_future());
+
+ /* wrap async FileSystem::SetReplication with promise to make it a blocking call */
+ auto h = [callstate](const Status &s) {
+ callstate->set_value(s);
+ };
+
+ SetReplication(path, replication, h);
+
+ /* block until promise is set */
+ auto returnstate = future.get();
+ Status stat = returnstate;
+
+ return stat;
+}
+
+Status FileSystemImpl::SetTimes(const std::string & path, uint64_t mtime, uint64_t atime) {
+ LOG_DEBUG(kFileSystem,
+ << "FileSystemImpl::[sync]SetTimes(" << FMT_THIS_ADDR << ", path=" << path <<
+ ", mtime=" << mtime << ", atime=" << atime << ") called");
+
+ auto callstate = std::make_shared<std::promise<Status>>();
+ std::future<Status> future(callstate->get_future());
+
+ /* wrap async FileSystem::SetTimes with promise to make it a blocking call */
+ auto h = [callstate](const Status &s) {
+ callstate->set_value(s);
+ };
+
+ SetTimes(path, mtime, atime, h);
+
+ /* block until promise is set */
+ auto returnstate = future.get();
+ Status stat = returnstate;
+
+ return stat;
+}
+
+Status FileSystemImpl::GetFileInfo(const std::string &path,
+ StatInfo & stat_info) {
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl::[sync]GetFileInfo("
+ << FMT_THIS_ADDR << ", path="
+ << path << ") called");
+
+ auto callstate = std::make_shared<std::promise<std::tuple<Status, StatInfo>>>();
+ std::future<std::tuple<Status, StatInfo>> future(callstate->get_future());
+
+ /* wrap async FileSystem::GetFileInfo with promise to make it a blocking call */
+ auto h = [callstate](const Status &s, const StatInfo &si) {
+ callstate->set_value(std::make_tuple(s, si));
+ };
+
+ GetFileInfo(path, h);
+
+ /* block until promise is set */
+ auto returnstate = future.get();
+ Status stat = std::get<0>(returnstate);
+ StatInfo info = std::get<1>(returnstate);
+
+ if (!stat.ok()) {
+ return stat;
+ }
+
+ stat_info = info;
+ return stat;
+}
+
+Status FileSystemImpl::GetFsStats(FsInfo & fs_info) {
+ LOG_DEBUG(kFileSystem,
+ << "FileSystemImpl::[sync]GetFsStats(" << FMT_THIS_ADDR << ") called");
+
+ auto callstate = std::make_shared<std::promise<std::tuple<Status, FsInfo>>>();
+ std::future<std::tuple<Status, FsInfo>> future(callstate->get_future());
+
+ /* wrap async FileSystem::GetFsStats with promise to make it a blocking call */
+ auto h = [callstate](const Status &s, const FsInfo &si) {
+ callstate->set_value(std::make_tuple(s, si));
+ };
+
+ GetFsStats(h);
+
+ /* block until promise is set */
+ auto returnstate = future.get();
+ Status stat = std::get<0>(returnstate);
+ FsInfo info = std::get<1>(returnstate);
+
+ if (!stat.ok()) {
+ return stat;
+ }
+
+ fs_info = info;
+ return stat;
+}
+
+Status FileSystemImpl::GetListing(const std::string &path, std::vector<StatInfo> * stat_infos) {
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl::[sync]GetListing("
+ << FMT_THIS_ADDR << ", path="
+ << path << ") called");
+
+ if (!stat_infos) {
+ return Status::InvalidArgument("FileSystemImpl::GetListing: argument 'stat_infos' cannot be NULL");
+ }
+
+ auto callstate = std::make_shared<std::promise<Status>>();
+ std::future<Status> future(callstate->get_future());
+
+ /* wrap async FileSystem::GetListing with promise to make it a blocking call.
+ *
+ Keep requesting more until we get the entire listing, and don't set the promise
+ * until we have the entire listing.
+ */
+ auto h = [callstate, stat_infos](const Status &s, const std::vector<StatInfo> & si, bool has_more) -> bool {
+ if (!si.empty()) {
+ stat_infos->insert(stat_infos->end(), si.begin(), si.end());
+ }
+
+ bool done = !s.ok() || !has_more;
+ if (done) {
+ callstate->set_value(s);
+ return false;
+ }
+ return true;
+ };
+
+ GetListing(path, h);
+
+ /* block until promise is set */
+ Status stat = future.get();
+
+ return stat;
+}
+
+Status FileSystemImpl::Mkdirs(const std::string & path, uint16_t permissions, bool createparent) {
+ LOG_DEBUG(kFileSystem,
+ << "FileSystemImpl::[sync]Mkdirs(" << FMT_THIS_ADDR << ", path=" << path <<
+ ", permissions=" << permissions << ", createparent=" << createparent << ") called");
+
+ auto callstate = std::make_shared<std::promise<Status>>();
+ std::future<Status> future(callstate->get_future());
+
+ /* wrap async FileSystem::Mkdirs with promise to make it a blocking call */
+ auto h = [callstate](const Status &s) {
+ callstate->set_value(s);
+ };
+
+ Mkdirs(path, permissions, createparent, h);
+
+ /* block until promise is set */
+ auto returnstate = future.get();
+ Status stat = returnstate;
+
+ return stat;
+}
+
+Status FileSystemImpl::Delete(const std::string &path, bool recursive) {
+ LOG_DEBUG(kFileSystem,
+ << "FileSystemImpl::[sync]Delete(" << FMT_THIS_ADDR << ", path=" << path << ", recursive=" << recursive << ") called");
+
+ auto callstate = std::make_shared<std::promise<Status>>();
+ std::future<Status> future(callstate->get_future());
+
+ /* wrap async FileSystem::Delete with promise to make it a blocking call */
+ auto h = [callstate](const Status &s) {
+ callstate->set_value(s);
+ };
+
+ Delete(path, recursive, h);
+
+ /* block until promise is set */
+ auto returnstate = future.get();
+ Status stat = returnstate;
+
+ return stat;
+}
+
+Status FileSystemImpl::Rename(const std::string &oldPath, const std::string &newPath) {
+ LOG_DEBUG(kFileSystem,
+ << "FileSystemImpl::[sync]Rename(" << FMT_THIS_ADDR << ", oldPath=" << oldPath << ", newPath=" << newPath << ") called");
+
+ auto callstate = std::make_shared<std::promise<Status>>();
+ std::future<Status> future(callstate->get_future());
+
+ /* wrap async FileSystem::Rename with promise to make it a blocking call */
+ auto h = [callstate](const Status &s) {
+ callstate->set_value(s);
+ };
+
+ Rename(oldPath, newPath, h);
+
+ /* block until promise is set */
+ auto returnstate = future.get();
+ Status stat = returnstate;
+
+ return stat;
+}
+
+Status FileSystemImpl::SetPermission(const std::string & path, uint16_t permissions) {
+ LOG_DEBUG(kFileSystem,
+ << "FileSystemImpl::[sync]SetPermission(" << FMT_THIS_ADDR << ", path=" << path << ", permissions=" << permissions << ") called");
+
+ auto callstate = std::make_shared<std::promise<Status>>();
+ std::future<Status> future(callstate->get_future());
+
+ /* wrap async FileSystem::SetPermission with promise to make it a blocking call */
+ auto h = [callstate](const Status &s) {
+ callstate->set_value(s);
+ };
+
+ SetPermission(path, permissions, h);
+
+ /* block until promise is set */
+ Status stat = future.get();
+
+ return stat;
+}
+
+Status FileSystemImpl::SetOwner(const std::string & path, const std::string & username,
+ const std::string & groupname) {
+ LOG_DEBUG(kFileSystem,
+ << "FileSystemImpl::[sync]SetOwner(" << FMT_THIS_ADDR << ", path=" << path << ", username=" << username << ", groupname=" << groupname << ") called");
+
+ auto callstate = std::make_shared<std::promise<Status>>();
+ std::future<Status> future(callstate->get_future());
+
+ /* wrap async FileSystem::SetOwner with promise to make it a blocking call */
+ auto h = [callstate](const Status &s) {
+ callstate->set_value(s);
+ };
+
+ SetOwner(path, username, groupname, h);
+
+ /* block until promise is set */
+ Status stat = future.get();
+ return stat;
+}
+
+Status FileSystemImpl::Find(const std::string &path, const std::string &name, const uint32_t maxdepth, std::vector<StatInfo> * stat_infos) {
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl::[sync]Find("
+ << FMT_THIS_ADDR << ", path="
+ << path << ", name="
+ << name << ") called");
+
+ if (!stat_infos) {
+ return Status::InvalidArgument("FileSystemImpl::Find: argument 'stat_infos' cannot be NULL");
+ }
+
+ // In this case, we're going to have the async code populate stat_infos.
+
+ std::promise<void> promise = std::promise<void>();
+ std::future<void> future(promise.get_future());
+ Status status = Status::OK();
+
+ /**
+ * Keep requesting more until we get the entire listing. Set the promise
+ * when we have the entire listing to stop.
+ *
+ * Find guarantees that the handler will only be called once at a time,
+ * so we do not need any locking here
+ */
+ auto h = [&status, &promise, stat_infos](const Status &s, const std::vector<StatInfo> & si, bool has_more_results) -> bool {
+ if (!si.empty()) {
+ stat_infos->insert(stat_infos->end(), si.begin(), si.end());
+ }
+ if (!s.ok() && status.ok()){
+ //We make sure we set 'status' only on the first error.
+ status = s;
+ }
+ if (!has_more_results) {
+ promise.set_value();
+ return false;
+ }
+ return true;
+ };
+
+ Find(path, name, maxdepth, h);
+
+ /* block until promise is set */
+ future.get();
+ return status;
+}
+
+Status FileSystemImpl::CreateSnapshot(const std::string &path,
+ const std::string &name) {
+ LOG_DEBUG(kFileSystem,
+ << "FileSystemImpl::[sync]CreateSnapshot(" << FMT_THIS_ADDR << ", path=" << path << ", name=" << name << ") called");
+
+ auto callstate = std::make_shared<std::promise<Status>>();
+ std::future<Status> future(callstate->get_future());
+
+ /* wrap async FileSystem::CreateSnapshot with promise to make it a blocking call */
+ auto h = [callstate](const Status &s) {
+ callstate->set_value(s);
+ };
+
+ CreateSnapshot(path, name, h);
+
+ /* block until promise is set */
+ auto returnstate = future.get();
+ Status stat = returnstate;
+
+ return stat;
+}
+
+Status FileSystemImpl::DeleteSnapshot(const std::string &path,
+ const std::string &name) {
+ LOG_DEBUG(kFileSystem,
+ << "FileSystemImpl::[sync]DeleteSnapshot(" << FMT_THIS_ADDR << ", path=" << path << ", name=" << name << ") called");
+
+ auto callstate = std::make_shared<std::promise<Status>>();
+ std::future<Status> future(callstate->get_future());
+
+ /* wrap async FileSystem::DeleteSnapshot with promise to make it a blocking call */
+ auto h = [callstate](const Status &s) {
+ callstate->set_value(s);
+ };
+
+ DeleteSnapshot(path, name, h);
+
+ /* block until promise is set */
+ auto returnstate = future.get();
+ Status stat = returnstate;
+
+ return stat;
+}
+
+Status FileSystemImpl::AllowSnapshot(const std::string &path) {
+ LOG_DEBUG(kFileSystem,
+ << "FileSystemImpl::[sync]AllowSnapshot(" << FMT_THIS_ADDR << ", path=" << path << ") called");
+
+ auto callstate = std::make_shared<std::promise<Status>>();
+ std::future<Status> future(callstate->get_future());
+
+ /* wrap async FileSystem::AllowSnapshot with promise to make it a blocking call */
+ auto h = [callstate](const Status &s) {
+ callstate->set_value(s);
+ };
+
+ AllowSnapshot(path, h);
+
+ /* block until promise is set */
+ auto returnstate = future.get();
+ Status stat = returnstate;
+
+ return stat;
+}
+
+Status FileSystemImpl::DisallowSnapshot(const std::string &path) {
+ LOG_DEBUG(kFileSystem,
+ << "FileSystemImpl::[sync]DisallowSnapshot(" << FMT_THIS_ADDR << ", path=" << path << ") called");
+
+ auto callstate = std::make_shared<std::promise<Status>>();
+ std::future<Status> future(callstate->get_future());
+
+ /* wrap async FileSystem::DisallowSnapshot with promise to make it a blocking call */
+ auto h = [callstate](const Status &s) {
+ callstate->set_value(s);
+ };
+
+ DisallowSnapshot(path, h);
+
+ /* block until promise is set */
+ auto returnstate = future.get();
+ Status stat = returnstate;
+
+ return stat;
+}
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7837c4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
index 032dfc8..3dd4aae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
@@ -72,7 +72,7 @@ target_link_libraries(remote_block_reader_test test_common reader proto common c
add_memcheck_test(remote_block_reader remote_block_reader_test)
add_executable(sasl_digest_md5_test sasl_digest_md5_test.cc)
-target_link_libraries(sasl_digest_md5_test common ${OPENSSL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(sasl_digest_md5_test common ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT})
add_memcheck_test(sasl_digest_md5 sasl_digest_md5_test)
add_executable(retry_policy_test retry_policy_test.cc)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7837c4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/sasl_digest_md5_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/sasl_digest_md5_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/sasl_digest_md5_test.cc
index 0797853..553ffa4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/sasl_digest_md5_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/sasl_digest_md5_test.cc
@@ -18,6 +18,7 @@
#include "common/sasl_authenticator.h"
#include <gtest/gtest.h>
+#include <google/protobuf/stubs/common.h>
namespace hdfs {
@@ -40,5 +41,7 @@ TEST(DigestMD5AuthenticatorTest, TestResponse) {
ASSERT_TRUE(status.ok());
ASSERT_TRUE(result.find("response=3a286c2c385b92a06ebc66d58b8c4330") !=
std::string::npos);
+
+ google::protobuf::ShutdownProtobufLibrary();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/52] [abbrv] hadoop git commit: HDFS-12104: libhdfs++: Make sure
all steps in SaslProtocol end up calling AuthComplete. Contributed by James
Clampffer.
Posted by jh...@apache.org.
HDFS-12104: libhdfs++: Make sure all steps in SaslProtocol end up calling AuthComplete. Contributed by James Clampffer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e64ba874
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e64ba874
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e64ba874
Branch: refs/heads/HDFS-8707
Commit: e64ba87486b9cb40474dfe27cf735cf179cb174c
Parents: 3c743b4
Author: James Clampffer <ja...@hp.com>
Authored: Mon Jul 10 12:09:30 2017 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e64ba874/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
index 49abe58..3d40c0f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
@@ -87,6 +87,7 @@ void SaslProtocol::Authenticate(std::function<void(const Status & status, const
// We cheat here since this is always called while holding the RpcConnection's lock
std::shared_ptr<RpcConnection> connection = connection_.lock();
if (!connection) {
+ AuthComplete(Status::AuthenticationFailed("Lost RPC Connection"), AuthInfo());
return;
}
@@ -335,6 +336,7 @@ bool SaslProtocol::SendSaslMessage(RpcSaslProto & message)
std::shared_ptr<RpcConnection> connection = connection_.lock();
if (!connection) {
LOG_DEBUG(kRPC, << "Tried sending a SASL Message but the RPC connection was gone");
+ AuthComplete(Status::AuthenticationFailed("Lost RPC Connection"), AuthInfo());
return false;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/52] [abbrv] hadoop git commit: Revert HDFS-10543 patch due to
correctness issues it caused. Done by James Clampffer
Posted by jh...@apache.org.
Revert HDFS-10543 patch due to correctness issues it caused. Done by James Clampffer
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d75c8356
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d75c8356
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d75c8356
Branch: refs/heads/HDFS-8707
Commit: d75c835696ac09eef173c4a90d65e5e300edf06a
Parents: a586ccb
Author: James <jh...@apache.org>
Authored: Wed Jul 27 17:02:11 2016 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../main/native/libhdfspp/lib/fs/filehandle.cc | 38 ++++++++------------
1 file changed, 15 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d75c8356/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
index df147d3..40f1b4a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
@@ -78,34 +78,26 @@ Status FileHandleImpl::PositionRead(void *buf, size_t *nbyte, off_t offset) {
LOG_TRACE(kFileHandle, << "FileHandleImpl::[sync]PositionRead("
<< FMT_THIS_ADDR << ", buf=" << buf
<< ", nbyte=" << *nbyte << ") called");
- size_t totalBytesRead = 0;
- Status stat = Status::OK();
- while (*nbyte != 0 && offset < (off_t)(file_info_->file_length_)) {
- auto callstate = std::make_shared<std::promise<std::tuple<Status, size_t>>>();
- std::future<std::tuple<Status, size_t>> future(callstate->get_future());
- /* wrap async call with promise/future to make it blocking */
- auto callback = [callstate](const Status &s, size_t bytes) {
- callstate->set_value(std::make_tuple(s,bytes));
- };
+ auto callstate = std::make_shared<std::promise<std::tuple<Status, size_t>>>();
+ std::future<std::tuple<Status, size_t>> future(callstate->get_future());
- PositionRead(buf, *nbyte, offset, callback);
+ /* wrap async call with promise/future to make it blocking */
+ auto callback = [callstate](const Status &s, size_t bytes) {
+ callstate->set_value(std::make_tuple(s,bytes));
+ };
- /* wait for async to finish */
- auto returnstate = future.get();
- stat = std::get<0>(returnstate);
+ PositionRead(buf, *nbyte, offset, callback);
- if (!stat.ok()) {
- return stat;
- }
+ /* wait for async to finish */
+ auto returnstate = future.get();
+ auto stat = std::get<0>(returnstate);
- size_t bytesRead = std::get<1>(returnstate);
- *nbyte = *nbyte - bytesRead;
- totalBytesRead += bytesRead;
- offset += bytesRead;
+ if (!stat.ok()) {
+ return stat;
}
- /* Update the bytes read for return */
- *nbyte = totalBytesRead;
+
+ *nbyte = std::get<1>(returnstate);
return stat;
}
@@ -118,8 +110,8 @@ Status FileHandleImpl::Read(void *buf, size_t *nbyte) {
if(!stat.ok()) {
return stat;
}
- offset_ += *nbyte;
+ offset_ += *nbyte;
return Status::OK();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/52] [abbrv] hadoop git commit: HDFS-10796: libhdfs++: rationalize
ioservice interactions. Contributed by James Clampffer.
Posted by jh...@apache.org.
HDFS-10796: libhdfs++: rationalize ioservice interactions. Contributed by James Clampffer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fbff671e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fbff671e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fbff671e
Branch: refs/heads/HDFS-8707
Commit: fbff671e0fae04b3ab8e11684de376f9c5c5f74a
Parents: dd7837c
Author: James <jh...@apache.org>
Authored: Fri Dec 9 18:06:06 2016 -0500
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../native/libhdfspp/include/hdfspp/hdfspp.h | 42 ++++++--
.../native/libhdfspp/include/hdfspp/options.h | 7 ++
.../libhdfspp/lib/common/hdfs_ioservice.cc | 99 ++++++++++++++++-
.../libhdfspp/lib/common/hdfs_ioservice.h | 41 +++++++-
.../main/native/libhdfspp/lib/common/logging.cc | 19 ++--
.../main/native/libhdfspp/lib/common/logging.h | 15 ++-
.../main/native/libhdfspp/lib/common/options.cc | 6 +-
.../main/native/libhdfspp/lib/fs/filesystem.cc | 105 ++++++++++++-------
.../main/native/libhdfspp/lib/fs/filesystem.h | 19 ++--
.../main/native/libhdfspp/tests/CMakeLists.txt | 5 +
.../native/libhdfspp/tests/hdfspp_mini_dfs.h | 8 +-
11 files changed, 292 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbff671e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
index 78ad594..5fbd3d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
@@ -39,24 +39,47 @@ namespace hdfs {
* When an operation is queued into an IoService, the IoService will
* run the callback handler associated with the operation. Note that
* the IoService must be stopped before destructing the objects that
- * file the operations.
+ * post the operations.
*
- * From an implementation point of view the IoService object wraps the
- * ::asio::io_service objects. Please see the related documentation
- * for more details.
+ * From an implementation point of view the hdfs::IoService provides
+ * a thin wrapper over an asio::io_service object so that additional
+ * instrumentation and functionality can be added.
**/
-class IoService {
+
+class IoService : public std::enable_shared_from_this<IoService>
+{
public:
static IoService *New();
+ static std::shared_ptr<IoService> MakeShared();
+ virtual ~IoService();
+
+ /**
+ * Start up as many threads as there are logical processors.
+ * Return number of threads created.
+ **/
+ virtual unsigned int InitDefaultWorkers() = 0;
+
+ /**
+ * Initialize with thread_count handler threads.
+ * If thread count is less than one print a log message and default to one thread.
+ * Return number of threads created.
+ **/
+ virtual unsigned int InitWorkers(unsigned int thread_count) = 0;
+
+ /**
+ * Place an item on the execution queue. Will be invoked from outside of the calling context.
+ **/
+ virtual void PostTask(std::function<void(void)>& asyncTask) = 0;
+
/**
* Run the asynchronous tasks associated with this IoService.
**/
virtual void Run() = 0;
/**
* Stop running asynchronous tasks associated with this IoService.
+ * All worker threads will return as soon as they finish executing their current task.
**/
virtual void Stop() = 0;
- virtual ~IoService();
};
/**
@@ -164,6 +187,13 @@ class FileSystem {
IoService *&io_service, const std::string &user_name, const Options &options);
/**
+ * Works the same as the other FileSystem::New but takes a copy of an existing IoService.
+ * The shared IoService is expected to already have worker threads initialized.
+ **/
+ static FileSystem *New(
+ std::shared_ptr<IoService>, const std::string &user_name, const Options &options);
+
+ /**
* Returns a new instance with default user and option, with the default IOService.
**/
static FileSystem *New();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbff671e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
index b73a729..456390f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
@@ -123,6 +123,13 @@ struct Options {
long block_size;
static const long kDefaultBlockSize = 128*1024*1024;
+ /**
+ * Asio worker thread count
+ * default: -1, indicates number of hardware threads
+ **/
+ int io_threads_;
+ static const int kDefaultIoThreads = -1;
+
Options();
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbff671e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.cc
index f6876cd..578b782 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.cc
@@ -18,14 +18,105 @@
#include "hdfs_ioservice.h"
+#include <thread>
+#include <mutex>
+#include <vector>
+
#include "common/logging.h"
namespace hdfs {
IoService::~IoService() {}
-IoService *IoService::New() { return new IoServiceImpl(); }
+IoService *IoService::New() {
+ return new IoServiceImpl();
+}
+
+std::shared_ptr<IoService> IoService::MakeShared() {
+ return std::make_shared<IoServiceImpl>();
+}
+
+
+unsigned int IoServiceImpl::InitDefaultWorkers() {
+ LOG_TRACE(kAsyncRuntime, << "IoServiceImpl::InitDefaultWorkers@" << this << " called.");
+ unsigned int logical_thread_count = std::thread::hardware_concurrency();
+#ifndef DISABLE_CONCURRENT_WORKERS
+ if(logical_thread_count < 1) {
+ LOG_WARN(kAsyncRuntime, << "IoServiceImpl::InitDefaultWorkers did not detect any logical processors. Defaulting to 1 worker thread.");
+ } else {
+ LOG_DEBUG(kRPC, << "IoServiceImpl::InitDefaultWorkers detected " << logical_thread_count << " logical threads and will spawn a worker for each.");
+ }
+#else
+ if(logical_thread_count > 0) {
+ LOG_DEBUG(kAsyncRuntime, << "IoServiceImpl::InitDefaultWorkers: " << logical_thread_count << " threads available. Concurrent workers are disabled so 1 worker thread will be used");
+ }
+ logical_thread_count = 1;
+#endif
+ return InitWorkers(logical_thread_count);
+}
+
+unsigned int IoServiceImpl::InitWorkers(unsigned int thread_count) {
+#ifdef DISABLED_CONCURRENT_WORKERS
+ LOG_DEBUG(kAsyncRuntime, << "IoServiceImpl::InitWorkers: " << thread_count << " threads specified but concurrent workers are disabled so 1 will be used");
+ thread_count = 1;
+#endif
+ unsigned int created_threads = 0;
+ for(unsigned int i=0; i<thread_count; i++) {
+ bool created = AddWorkerThread();
+ if(created) {
+ created_threads++;
+ } else {
+ LOG_DEBUG(kAsyncRuntime, << "IoServiceImpl@" << this << " ::InitWorkers failed to create a worker thread");
+ }
+ }
+ if(created_threads != thread_count) {
+ LOG_WARN(kAsyncRuntime, << "IoServiceImpl@" << this << " ::InitWorkers attempted to create "
+ << thread_count << " but only created " << created_threads
+ << " worker threads. Make sure this process has adequate resources.");
+ }
+ return created_threads;
+}
+bool IoServiceImpl::AddWorkerThread() {
+ mutex_guard state_lock(state_lock_);
+ auto async_worker = [this]() {
+ this->ThreadStartHook();
+ this->Run();
+ this->ThreadExitHook();
+ };
+ worker_threads_.push_back(WorkerPtr( new std::thread(async_worker)) );
+ return true;
+}
+
+
+void IoServiceImpl::ThreadStartHook() {
+ mutex_guard state_lock(state_lock_);
+ LOG_DEBUG(kAsyncRuntime, << "Worker thread #" << std::this_thread::get_id() << " for IoServiceImpl@" << this << " starting");
+}
+
+void IoServiceImpl::ThreadExitHook() {
+ mutex_guard state_lock(state_lock_);
+ LOG_DEBUG(kAsyncRuntime, << "Worker thread #" << std::this_thread::get_id() << " for IoServiceImpl@" << this << " exiting");
+}
+
+void IoServiceImpl::PostTask(std::function<void(void)>& asyncTask) {
+ io_service_.post(asyncTask);
+}
+
+void IoServiceImpl::WorkerDeleter::operator()(std::thread *t) {
+ // It is far too easy to destroy the filesystem (and thus the threadpool)
+ // from within one of the worker threads, leading to a deadlock. Let's
+ // provide some explicit protection.
+ if(t->get_id() == std::this_thread::get_id()) {
+ LOG_ERROR(kAsyncRuntime, << "FileSystemImpl::WorkerDeleter::operator(treadptr="
+ << t << ") : FATAL: Attempted to destroy a thread pool"
+ "from within a callback of the thread pool!");
+ }
+ t->join();
+ delete t;
+}
+
+// As long as this just forwards to an asio::io_service method it doesn't need a lock
void IoServiceImpl::Run() {
// The IoService executes callbacks provided by library users in the context of worker threads,
// there is no way of preventing those callbacks from throwing but we can at least prevent them
@@ -33,7 +124,7 @@ void IoServiceImpl::Run() {
// As recommended in http://www.boost.org/doc/libs/1_39_0/doc/html/boost_asio/reference/io_service.html#boost_asio.reference.io_service.effect_of_exceptions_thrown_from_handlers
asio::io_service::work work(io_service_);
- for(;;)
+ while(true)
{
try
{
@@ -47,5 +138,9 @@ void IoServiceImpl::Run() {
}
}
+unsigned int IoServiceImpl::get_worker_thread_count() {
+ mutex_guard state_lock(state_lock_);
+ return worker_threads_.size();
+}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbff671e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.h
index 73d167e..294252b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.h
@@ -22,23 +22,56 @@
#include "hdfspp/hdfspp.h"
#include <asio/io_service.hpp>
+#include "common/util.h"
+
+#include <mutex>
+#include <thread>
namespace hdfs {
+// Uncomment this to determine if issues are due to concurrency or logic faults
+// If tests still fail with concurrency disabled it's most likely a logic bug
+#define DISABLE_CONCURRENT_WORKERS
+
/*
- * A thin wrapper over the asio::io_service.
- * -In the future this could own the worker threads that execute io tasks which
- * makes it easier to share IoServices between FileSystems. See HDFS-10796 for
- * rationale.
+ * A thin wrapper over the asio::io_service with a few extras
+ * -manages it's own worker threads
+ * -some helpers for sharing with multiple modules that need to do async work
*/
class IoServiceImpl : public IoService {
public:
+ IoServiceImpl() {}
+
+ virtual unsigned int InitDefaultWorkers() override;
+ virtual unsigned int InitWorkers(unsigned int thread_count) override;
+ virtual void PostTask(std::function<void(void)>& asyncTask) override;
virtual void Run() override;
virtual void Stop() override { io_service_.stop(); }
+
+ // Add a single worker thread, in the common case try to avoid this in favor
+ // of Init[Default]Workers. Public for use by tests and rare cases where a
+ // client wants very explicit control of threading for performance reasons
+ // e.g. pinning threads to NUMA nodes.
+ bool AddWorkerThread();
+
+ // Be very careful about using this: HDFS-10241
::asio::io_service &io_service() { return io_service_; }
+ unsigned int get_worker_thread_count();
private:
+ std::mutex state_lock_;
::asio::io_service io_service_;
+
+ // For doing logging + resource manager updates on thread start/exit
+ void ThreadStartHook();
+ void ThreadExitHook();
+
+ // Support for async worker threads
+ struct WorkerDeleter {
+ void operator()(std::thread *t);
+ };
+ typedef std::unique_ptr<std::thread, WorkerDeleter> WorkerPtr;
+ std::vector<WorkerPtr> worker_threads_;
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbff671e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.cc
index ac1c336..94bce83 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.cc
@@ -183,6 +183,11 @@ LogMessage& LogMessage::operator<<(void *ptr) {
}
+LogMessage& LogMessage::operator<<(const std::thread::id& tid) {
+ msg_buffer_ << tid;
+ return *this;
+}
+
std::string LogMessage::MsgString() const {
return msg_buffer_.str();
}
@@ -199,12 +204,13 @@ const char * LogMessage::level_string() const {
return kLevelStrings[level_];
}
-const char * kComponentStrings[5] = {
- "[Unknown ]",
- "[RPC ]",
- "[BlockReader ]",
- "[FileHandle ]",
- "[FileSystem ]"
+const char * kComponentStrings[6] = {
+ "[Unknown ]",
+ "[RPC ]",
+ "[BlockReader ]",
+ "[FileHandle ]",
+ "[FileSystem ]",
+ "[Async Runtime ]",
};
const char * LogMessage::component_string() const {
@@ -213,6 +219,7 @@ const char * LogMessage::component_string() const {
case kBlockReader: return kComponentStrings[2];
case kFileHandle: return kComponentStrings[3];
case kFileSystem: return kComponentStrings[4];
+ case kAsyncRuntime: return kComponentStrings[5];
default: return kComponentStrings[0];
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbff671e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h
index f807fc4..69f9c6e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h
@@ -27,6 +27,7 @@
#include <sstream>
#include <mutex>
#include <memory>
+#include <thread>
#include <asio/ip/tcp.hpp>
@@ -49,11 +50,12 @@ enum LogLevel {
};
enum LogSourceComponent {
- kUnknown = 1 << 0,
- kRPC = 1 << 1,
- kBlockReader = 1 << 2,
- kFileHandle = 1 << 3,
- kFileSystem = 1 << 4,
+ kUnknown = 1 << 0,
+ kRPC = 1 << 1,
+ kBlockReader = 1 << 2,
+ kFileHandle = 1 << 3,
+ kFileSystem = 1 << 4,
+ kAsyncRuntime = 1 << 5,
};
#define LOG_TRACE(C, MSG) do { \
@@ -196,6 +198,9 @@ class LogMessage {
//asio types
LogMessage& operator<<(const ::asio::ip::tcp::endpoint& endpoint);
+ //thread and mutex types
+ LogMessage& operator<<(const std::thread::id& tid);
+
std::string MsgString() const;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbff671e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc
index a889be5..728082e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc
@@ -39,7 +39,11 @@ Options::Options() : rpc_timeout(kDefaultRpcTimeout),
failover_max_retries(kDefaultFailoverMaxRetries),
failover_connection_max_retries(kDefaultFailoverConnectionMaxRetries),
authentication(kDefaultAuthentication),
- block_size(kDefaultBlockSize) {}
+ block_size(kDefaultBlockSize),
+ io_threads_(kDefaultIoThreads)
+{
+
+}
std::string NamenodeInfo::get_host() const {
return uri.get_host();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbff671e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
index d805716..a5f3aad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
@@ -79,11 +79,20 @@ FileSystem *FileSystem::New(
return new FileSystemImpl(io_service, user_name, options);
}
+FileSystem *FileSystem::New(
+ std::shared_ptr<IoService> io_service, const std::string &user_name, const Options &options) {
+ return new FileSystemImpl(io_service, user_name, options);
+}
+
FileSystem *FileSystem::New() {
// No, this pointer won't be leaked. The FileSystem takes ownership.
- IoService *io_service = IoService::New();
+ std::shared_ptr<IoService> io_service = IoService::MakeShared();
if(!io_service)
return nullptr;
+ int thread_count = io_service->InitDefaultWorkers();
+ if(thread_count < 1)
+ return nullptr;
+
std::string user_name = get_effective_user_name("");
Options options;
return new FileSystemImpl(io_service, user_name, options);
@@ -123,25 +132,56 @@ const std::string get_effective_user_name(const std::string &user_name) {
}
FileSystemImpl::FileSystemImpl(IoService *&io_service, const std::string &user_name, const Options &options) :
- options_(options), client_name_(GetRandomClientName()), io_service_(
- static_cast<IoServiceImpl *>(io_service)),
- nn_(
- &io_service_->io_service(), options, client_name_,
- get_effective_user_name(user_name), kNamenodeProtocol,
- kNamenodeProtocolVersion
- ), bad_node_tracker_(std::make_shared<BadDataNodeTracker>()),
- event_handlers_(std::make_shared<LibhdfsEvents>()) {
-
- LOG_TRACE(kFileSystem, << "FileSystemImpl::FileSystemImpl("
+ io_service_(static_cast<IoServiceImpl *>(io_service)), options_(options),
+ client_name_(GetRandomClientName()),
+ nn_(
+ &io_service_->io_service(), options, client_name_,
+ get_effective_user_name(user_name), kNamenodeProtocol,
+ kNamenodeProtocolVersion
+ ),
+ bad_node_tracker_(std::make_shared<BadDataNodeTracker>()),
+ event_handlers_(std::make_shared<LibhdfsEvents>())
+{
+
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl::FileSystemImpl("
<< FMT_THIS_ADDR << ") called");
// Poor man's move
io_service = nullptr;
- /* spawn background threads for asio delegation */
- unsigned int threads = 1 /* options.io_threads_, pending HDFS-9117 */;
- for (unsigned int i = 0; i < threads; i++) {
- AddWorkerThread();
+ unsigned int running_workers = 0;
+ if(options.io_threads_ < 1) {
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl::FileSystemImpl Initializing default number of worker threads");
+ running_workers = io_service_->InitDefaultWorkers();
+ } else {
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl::FileSystenImpl Initializing " << options_.io_threads_ << " worker threads.");
+ running_workers = io_service->InitWorkers(options_.io_threads_);
+ }
+
+ if(running_workers < 1) {
+ LOG_WARN(kFileSystem, << "FileSystemImpl::FileSystemImpl was unable to start worker threads");
+ }
+}
+
+FileSystemImpl::FileSystemImpl(std::shared_ptr<IoService> io_service, const std::string& user_name, const Options &options) :
+ io_service_(std::static_pointer_cast<IoServiceImpl>(io_service)), options_(options),
+ client_name_(GetRandomClientName()),
+ nn_(
+ &io_service_->io_service(), options, client_name_,
+ get_effective_user_name(user_name), kNamenodeProtocol,
+ kNamenodeProtocolVersion
+ ),
+ bad_node_tracker_(std::make_shared<BadDataNodeTracker>()),
+ event_handlers_(std::make_shared<LibhdfsEvents>())
+{
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl::FileSystemImpl("
+ << FMT_THIS_ADDR << ", shared IoService@" << io_service_.get() << ") called");
+ int worker_thread_count = io_service_->get_worker_thread_count();
+ if(worker_thread_count < 1) {
+ LOG_WARN(kFileSystem, << "FileSystemImpl::FileSystemImpl IoService provided doesn't have any worker threads. "
+ << "It needs at least 1 worker to connect to an HDFS cluster.")
+ } else {
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl::FileSystemImpl using " << worker_thread_count << " worker threads.");
}
}
@@ -154,7 +194,6 @@ FileSystemImpl::~FileSystemImpl() {
* Once worker threads are joined and deleted the service can be deleted.
**/
io_service_->Stop();
- worker_threads_.clear();
}
void FileSystemImpl::Connect(const std::string &server,
@@ -230,12 +269,21 @@ void FileSystemImpl::ConnectToDefaultFs(const std::function<void(const Status &,
int FileSystemImpl::AddWorkerThread() {
LOG_DEBUG(kFileSystem, << "FileSystemImpl::AddWorkerThread("
<< FMT_THIS_ADDR << ") called."
- << " Existing thread count = " << worker_threads_.size());
+ << " Existing thread count = " << WorkerThreadCount());
+
+ if(!io_service_)
+ return -1;
- auto service_task = [](IoService *service) { service->Run(); };
- worker_threads_.push_back(
- WorkerPtr(new std::thread(service_task, io_service_.get())));
- return worker_threads_.size();
+ io_service_->AddWorkerThread();
+ return 1;
+}
+
+int FileSystemImpl::WorkerThreadCount() {
+ if(!io_service_) {
+ return -1;
+ } else {
+ return io_service_->get_worker_thread_count();
+ }
}
void FileSystemImpl::Open(
@@ -714,21 +762,6 @@ void FileSystemImpl::DisallowSnapshot(const std::string &path,
nn_.DisallowSnapshot(path, handler);
}
-
-void FileSystemImpl::WorkerDeleter::operator()(std::thread *t) {
- // It is far too easy to destroy the filesystem (and thus the threadpool)
- // from within one of the worker threads, leading to a deadlock. Let's
- // provide some explicit protection.
- if(t->get_id() == std::this_thread::get_id()) {
- LOG_ERROR(kFileSystem, << "FileSystemImpl::WorkerDeleter::operator(treadptr="
- << t << ") : FATAL: Attempted to destroy a thread pool"
- "from within a callback of the thread pool!");
- }
- t->join();
- delete t;
-}
-
-
void FileSystemImpl::SetFsEventCallback(fs_event_callback callback) {
if (event_handlers_) {
event_handlers_->set_fs_callback(callback);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbff671e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
index 0e9cedd..80978cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
@@ -46,7 +46,8 @@ namespace hdfs {
class FileSystemImpl : public FileSystem {
public:
MEMCHECKED_CLASS(FileSystemImpl)
- FileSystemImpl(IoService *&io_service, const std::string& user_name, const Options &options);
+ explicit FileSystemImpl(IoService *&io_service, const std::string& user_name, const Options &options);
+ explicit FileSystemImpl(std::shared_ptr<IoService>, const std::string& user_name, const Options &options);
~FileSystemImpl() override;
/* attempt to connect to namenode, return bad status on failure */
@@ -176,7 +177,7 @@ public:
int AddWorkerThread();
/* how many worker threads are servicing asio requests */
- int WorkerThreadCount() { return worker_threads_.size(); }
+ int WorkerThreadCount();
/* all monitored events will need to lookup handlers */
std::shared_ptr<LibhdfsEvents> get_event_handlers();
@@ -184,24 +185,18 @@ public:
Options get_options();
private:
- const Options options_;
- const std::string client_name_;
- std::string cluster_name_;
/**
* The IoService must be the first member variable to ensure that it gets
* destroyed last. This allows other members to dequeue things from the
* service in their own destructors.
**/
- std::unique_ptr<IoServiceImpl> io_service_;
+ std::shared_ptr<IoServiceImpl> io_service_;
+ const Options options_;
+ const std::string client_name_;
+ std::string cluster_name_;
NameNodeOperations nn_;
std::shared_ptr<BadDataNodeTracker> bad_node_tracker_;
- struct WorkerDeleter {
- void operator()(std::thread *t);
- };
- typedef std::unique_ptr<std::thread, WorkerDeleter> WorkerPtr;
- std::vector<WorkerPtr> worker_threads_;
-
/**
* Runtime event monitoring handlers.
* Note: This is really handy to have for advanced usage but
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbff671e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
index 3dd4aae..395fad5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
@@ -112,6 +112,11 @@ add_executable(logging_test logging_test.cc)
target_link_libraries(logging_test common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT})
add_memcheck_test(logging_test logging_test)
+add_executable(hdfs_ioservice_test hdfs_ioservice_test.cc)
+target_link_libraries(hdfs_ioservice_test fs gmock_main common ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
+add_memcheck_test(hdfs_ioservice hdfs_ioservice_test)
+
+
#
#
# INTEGRATION TESTS - TESTS THE FULL LIBRARY AGAINST ACTUAL SERVERS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbff671e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfspp_mini_dfs.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfspp_mini_dfs.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfspp_mini_dfs.h
index 3ec58e1..aecced1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfspp_mini_dfs.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfspp_mini_dfs.h
@@ -113,7 +113,7 @@ private:
class MiniCluster {
public:
- MiniCluster() : io_service(IoService::New()) {
+ MiniCluster() : io_service(IoService::MakeShared()) {
struct NativeMiniDfsConf conf = {
1, /* doFormat */
0, /* webhdfs */
@@ -137,6 +137,10 @@ public:
// Connect via the C++ API
FSHandle connect(const std::string username) {
Options options;
+
+ unsigned int worker_count = io_service->InitDefaultWorkers();
+ EXPECT_NE(0, worker_count);
+
FileSystem * fs = FileSystem::New(io_service, username, options);
EXPECT_NE(nullptr, fs);
FSHandle result(fs);
@@ -184,7 +188,7 @@ public:
protected:
struct NativeMiniDfsCluster* clusterInfo;
- IoService * io_service;
+ std::shared_ptr<IoService> io_service;
};
} // namespace
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/52] [abbrv] hadoop git commit: HDFS-10739: libhdfs++: In
RpcEngine replace vector with deque for pending requests. Contributed by
Anatoli Shein.
Posted by jh...@apache.org.
HDFS-10739: libhdfs++: In RpcEngine replace vector with deque for pending requests. Contributed by Anatoli Shein.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b1ed72e0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b1ed72e0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b1ed72e0
Branch: refs/heads/HDFS-8707
Commit: b1ed72e098069efce4df5d20cd53613fdfc38842
Parents: 649aff1
Author: James <jh...@apache.org>
Authored: Tue Aug 9 15:55:18 2016 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../src/main/native/libhdfspp/lib/rpc/rpc_connection.cc | 1 -
.../src/main/native/libhdfspp/lib/rpc/rpc_engine.h | 5 +++--
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1ed72e0/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
index 03f83f4..fccfb09 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
@@ -445,7 +445,6 @@ void RpcConnection::SendRpcRequests(const std::vector<std::shared_ptr<Request> >
Status status = Status::ResourceUnavailable("RpcConnection closed before send.");
engine_->AsyncRpcCommsError(status, shared_from_this(), requests);
} else {
- pending_requests_.reserve(pending_requests_.size() + requests.size());
for (auto r: requests) {
if (r->method_name() != SASL_METHOD_NAME)
pending_requests_.push_back(r);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1ed72e0/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
index 47618a6..efaf407 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
@@ -41,6 +41,7 @@
#include <memory>
#include <unordered_map>
#include <vector>
+#include <deque>
#include <mutex>
#include <future>
@@ -234,10 +235,10 @@ class RpcConnection : public std::enable_shared_from_this<RpcConnection> {
// The request being sent over the wire; will also be in requests_on_fly_
std::shared_ptr<Request> request_over_the_wire_;
// Requests to be sent over the wire
- std::vector<std::shared_ptr<Request>> pending_requests_;
+ std::deque<std::shared_ptr<Request>> pending_requests_;
// Requests to be sent over the wire during authentication; not retried if
// there is a connection error
- std::vector<std::shared_ptr<Request>> auth_requests_;
+ std::deque<std::shared_ptr<Request>> auth_requests_;
// Requests that are waiting for responses
typedef std::unordered_map<int, std::shared_ptr<Request>> RequestOnFlyMap;
RequestOnFlyMap requests_on_fly_;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/52] [abbrv] hadoop git commit: HDFS-11544: libhdfs++: Improve C
API error reporting. Contributed by James Clampffer.
Posted by jh...@apache.org.
HDFS-11544: libhdfs++: Improve C API error reporting. Contributed by James Clampffer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fdb88eb5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fdb88eb5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fdb88eb5
Branch: refs/heads/HDFS-8707
Commit: fdb88eb57e18273d31681d31202320da0b0ca61e
Parents: 3e53da2
Author: James <jh...@apache.org>
Authored: Tue Mar 21 12:37:13 2017 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../native/libhdfspp/lib/bindings/c/hdfs.cc | 800 ++++++++++---------
1 file changed, 402 insertions(+), 398 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdb88eb5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
index 0608244..2856b0d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
@@ -41,6 +41,23 @@ using namespace std::placeholders;
static constexpr tPort kDefaultPort = 8020;
+// Functions exposed through the C API and extended C API get tagged with no-op
+// defines. This is to make it really clear what's being exposed in a way that
+// won't blend into the comments. Can also be used to add attributes or disable
+// the extensions e.g. "#define HDFS_EXT_API static"
+// API declared in hdfs.h shared with libhdfs
+#define HDFS_API
+// API extension functions specific to libhdfs++ declared in hdfs_ext.h
+#define HDFS_EXT_API
+
+// Common catch-all case to get exception and return the appropriate error code
+#define CATCH_AND_REPORT catch (const std::exception & e) { return ReportException(e); } \
+ catch (...) { return ReportCaughtNonException(); }
+
+
+
+
+
/* Separate the handles used by the C api from the C++ API*/
struct hdfs_internal {
hdfs_internal(FileSystem *p) : filesystem_(p), working_directory_("/") {}
@@ -79,9 +96,14 @@ struct hdfsFile_internal {
thread_local std::string errstr;
/* Fetch last error that happened in this thread */
+HDFS_EXT_API
int hdfsGetLastError(char *buf, int len) {
//No error message
if(errstr.empty()){
+ if(len > 0) {
+ // Null terminator if there is space
+ *buf = 0;
+ }
return -1;
}
@@ -104,6 +126,17 @@ int hdfsGetLastError(char *buf, int len) {
return 0;
}
+/* Clear out last error. This shouldn't need to be public if functions do it themselves */
+static void resetErrorForApiCall(const char *location = nullptr) {
+ // POSIX spec says caller should be responsible for resetting errno.
+ // Clearing it here just in case.
+ errno = 0;
+ if(location)
+ errstr = std::string(location) + " has not set an error message";
+ else
+ errstr.clear();
+}
+
/* Event callbacks for next open calls */
thread_local std::experimental::optional<fs_event_callback> fsEventCallback;
thread_local std::experimental::optional<file_event_callback> fileEventCallback;
@@ -255,7 +288,9 @@ optional<std::string> getAbsolutePath(hdfsFS fs, const char* path) {
* C API implementations
**/
+HDFS_API
int hdfsFileIsOpenForRead(hdfsFile file) {
+ resetErrorForApiCall("hdfsFileIsOpenForRead");
/* files can only be open for reads at the moment, do a quick check */
if (!CheckHandle(file)){
return 0;
@@ -263,7 +298,9 @@ int hdfsFileIsOpenForRead(hdfsFile file) {
return 1; // Update implementation when we get file writing
}
+HDFS_API
int hdfsFileIsOpenForWrite(hdfsFile file) {
+ resetErrorForApiCall("hdfsFileIsOpenForWrite");
/* files can only be open for reads at the moment, so return false */
CheckHandle(file);
return -1; // Update implementation when we get file writing
@@ -276,11 +313,7 @@ int hdfsConfGetLong(const char *key, int64_t *val)
errno = 0;
hdfsBuilder builder;
return hdfsBuilderConfGetLong(&builder, key, val);
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
- }
+ } CATCH_AND_REPORT
}
hdfsFS doHdfsConnect(optional<std::string> nn, optional<tPort> port, optional<std::string> user, const Options & options) {
@@ -332,11 +365,12 @@ hdfsFS doHdfsConnect(optional<std::string> nn, optional<tPort> port, optional<st
}
}
+HDFS_EXT_API
hdfsFS hdfsAllocateFileSystem(struct hdfsBuilder *bld) {
+ resetErrorForApiCall("hdfsAllocateFileSystem");
// Same idea as the first half of doHdfsConnect, but return the wrapped FS before
// connecting.
try {
- errno = 0;
std::shared_ptr<IoService> io_service = IoService::MakeShared();
int io_thread_count = bld->config.GetOptions().io_threads_;
@@ -367,7 +401,9 @@ hdfsFS hdfsAllocateFileSystem(struct hdfsBuilder *bld) {
return nullptr;
}
+HDFS_EXT_API
int hdfsConnectAllocated(hdfsFS fs, struct hdfsBuilder *bld) {
+ resetErrorForApiCall("hdfsConnectAllocated");
if(!CheckSystem(fs)) {
return ENODEV;
}
@@ -420,25 +456,35 @@ int hdfsConnectAllocated(hdfsFS fs, struct hdfsBuilder *bld) {
return 0;
}
+HDFS_API
hdfsFS hdfsConnect(const char *nn, tPort port) {
- return hdfsConnectAsUser(nn, port, "");
+ resetErrorForApiCall("hdfsConnect");
+ return doHdfsConnect(std::string(nn), port, std::string(""), Options());
}
+HDFS_API
hdfsFS hdfsConnectAsUser(const char* nn, tPort port, const char *user) {
+ resetErrorForApiCall("hdfsConnectAsUser");
return doHdfsConnect(std::string(nn), port, std::string(user), Options());
}
+HDFS_API
hdfsFS hdfsConnectAsUserNewInstance(const char* nn, tPort port, const char *user ) {
+ resetErrorForApiCall("hdfsConnectAsUserNewInstance");
//libhdfspp always returns a new instance
return doHdfsConnect(std::string(nn), port, std::string(user), Options());
}
+HDFS_API
hdfsFS hdfsConnectNewInstance(const char* nn, tPort port) {
+ resetErrorForApiCall("hdfsConnectNewInstance");
//libhdfspp always returns a new instance
- return hdfsConnectAsUser(nn, port, "");
+ return doHdfsConnect(std::string(nn), port, std::string(""), Options());
}
+HDFS_EXT_API
int hdfsCancelPendingConnection(hdfsFS fs) {
+ resetErrorForApiCall("hdfsCancelPendingConnection");
// todo: stick an enum in hdfs_internal to check the connect state
if(!CheckSystem(fs)) {
return ENODEV;
@@ -458,10 +504,11 @@ int hdfsCancelPendingConnection(hdfsFS fs) {
}
}
+HDFS_API
int hdfsDisconnect(hdfsFS fs) {
+ resetErrorForApiCall("hdfsDisconnect");
try
{
- errno = 0;
if (!fs) {
ReportError(ENODEV, "Cannot disconnect null FS handle.");
return -1;
@@ -469,18 +516,15 @@ int hdfsDisconnect(hdfsFS fs) {
delete fs;
return 0;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
- }
+ } CATCH_AND_REPORT
}
+HDFS_API
hdfsFile hdfsOpenFile(hdfsFS fs, const char *path, int flags, int bufferSize,
short replication, tSize blocksize) {
+ resetErrorForApiCall("hdfsOpenFile");
try
{
- errno = 0;
(void)flags;
(void)bufferSize;
(void)replication;
@@ -512,26 +556,24 @@ hdfsFile hdfsOpenFile(hdfsFS fs, const char *path, int flags, int bufferSize,
}
}
+HDFS_API
int hdfsCloseFile(hdfsFS fs, hdfsFile file) {
+ resetErrorForApiCall("hdfsCloseFile");
try
{
- errno = 0;
if (!CheckSystemAndHandle(fs, file)) {
return -1;
}
delete file;
return 0;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
- }
+ } CATCH_AND_REPORT
}
+HDFS_API
char* hdfsGetWorkingDirectory(hdfsFS fs, char *buffer, size_t bufferSize) {
+ resetErrorForApiCall("hdfsGetWorkingDirectory");
try
{
- errno = 0;
if (!CheckSystem(fs)) {
return nullptr;
}
@@ -556,10 +598,11 @@ char* hdfsGetWorkingDirectory(hdfsFS fs, char *buffer, size_t bufferSize) {
}
}
+HDFS_API
int hdfsSetWorkingDirectory(hdfsFS fs, const char* path) {
+ resetErrorForApiCall("hdfsSetWorkingDirectory");
try
{
- errno = 0;
if (!CheckSystem(fs)) {
return -1;
}
@@ -575,25 +618,23 @@ int hdfsSetWorkingDirectory(hdfsFS fs, const char* path) {
}
fs->set_working_directory(withSlash);
return 0;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
- }
+ } CATCH_AND_REPORT
}
+HDFS_API
int hdfsAvailable(hdfsFS fs, hdfsFile file) {
+ resetErrorForApiCall("hdfsAvailable");
//Since we do not have read ahead implemented, return 0 if fs and file are good;
- errno = 0;
if (!CheckSystemAndHandle(fs, file)) {
return -1;
}
return 0;
}
+HDFS_API
tOffset hdfsGetDefaultBlockSize(hdfsFS fs) {
+ resetErrorForApiCall("hdfsGetDefaultBlockSize");
try {
- errno = 0;
return fs->get_impl()->get_options().block_size;
} catch (const std::exception & e) {
ReportException(e);
@@ -604,9 +645,10 @@ tOffset hdfsGetDefaultBlockSize(hdfsFS fs) {
}
}
+HDFS_API
tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path) {
+ resetErrorForApiCall("hdfsGetDefaultBlockSizeAtPath");
try {
- errno = 0;
if (!CheckSystem(fs)) {
return -1;
}
@@ -633,58 +675,53 @@ tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path) {
}
}
+HDFS_API
int hdfsSetReplication(hdfsFS fs, const char* path, int16_t replication) {
- try {
- errno = 0;
- if (!CheckSystem(fs)) {
- return -1;
- }
- const optional<std::string> abs_path = getAbsolutePath(fs, path);
- if(!abs_path) {
- return -1;
- }
- if(replication < 1){
- return Error(Status::InvalidArgument("SetReplication: argument 'replication' cannot be less than 1"));
- }
- Status stat;
- stat = fs->get_impl()->SetReplication(*abs_path, replication);
- if (!stat.ok()) {
- return Error(stat);
- }
- return 0;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
+ resetErrorForApiCall("hdfsSetReplication");
+ try {
+ if (!CheckSystem(fs)) {
+ return -1;
+ }
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
+ }
+ if(replication < 1){
+ return Error(Status::InvalidArgument("SetReplication: argument 'replication' cannot be less than 1"));
+ }
+ Status stat;
+ stat = fs->get_impl()->SetReplication(*abs_path, replication);
+ if (!stat.ok()) {
+ return Error(stat);
}
+ return 0;
+ } CATCH_AND_REPORT
}
+HDFS_API
int hdfsUtime(hdfsFS fs, const char* path, tTime mtime, tTime atime) {
- try {
- errno = 0;
- if (!CheckSystem(fs)) {
- return -1;
- }
- const optional<std::string> abs_path = getAbsolutePath(fs, path);
- if(!abs_path) {
- return -1;
- }
- Status stat;
- stat = fs->get_impl()->SetTimes(*abs_path, mtime, atime);
- if (!stat.ok()) {
- return Error(stat);
- }
- return 0;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
+ resetErrorForApiCall("hdfsUtime");
+ try {
+ if (!CheckSystem(fs)) {
+ return -1;
}
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
+ }
+ Status stat;
+ stat = fs->get_impl()->SetTimes(*abs_path, mtime, atime);
+ if (!stat.ok()) {
+ return Error(stat);
+ }
+ return 0;
+ } CATCH_AND_REPORT
}
+HDFS_API
tOffset hdfsGetCapacity(hdfsFS fs) {
+ resetErrorForApiCall("hdfsGetCapacity");
try {
- errno = 0;
if (!CheckSystem(fs)) {
return -1;
}
@@ -705,9 +742,10 @@ tOffset hdfsGetCapacity(hdfsFS fs) {
}
}
+HDFS_API
tOffset hdfsGetUsed(hdfsFS fs) {
+ resetErrorForApiCall("hdfsGetUsed");
try {
- errno = 0;
if (!CheckSystem(fs)) {
return -1;
}
@@ -777,9 +815,10 @@ void StatInfoToHdfsFileInfo(hdfsFileInfo * file_info,
file_info->mLastAccess = stat_info.access_time;
}
+HDFS_API
int hdfsExists(hdfsFS fs, const char *path) {
+ resetErrorForApiCall("hdfsExists");
try {
- errno = 0;
if (!CheckSystem(fs)) {
return -1;
}
@@ -793,16 +832,13 @@ int hdfsExists(hdfsFS fs, const char *path) {
return Error(stat);
}
return 0;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
- }
+ } CATCH_AND_REPORT
}
+HDFS_API
hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char* path) {
+ resetErrorForApiCall("hdfsGetPathInfo");
try {
- errno = 0;
if (!CheckSystem(fs)) {
return nullptr;
}
@@ -828,61 +864,64 @@ hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char* path) {
}
}
+HDFS_API
hdfsFileInfo *hdfsListDirectory(hdfsFS fs, const char* path, int *numEntries) {
+ resetErrorForApiCall("hdfsListDirectory");
try {
- errno = 0;
- if (!CheckSystem(fs)) {
- *numEntries = 0;
- return nullptr;
- }
- const optional<std::string> abs_path = getAbsolutePath(fs, path);
- if(!abs_path) {
- return nullptr;
- }
- std::vector<StatInfo> stat_infos;
- Status stat = fs->get_impl()->GetListing(*abs_path, &stat_infos);
- if (!stat.ok()) {
- Error(stat);
- *numEntries = 0;
- return nullptr;
- }
- if(stat_infos.empty()){
- *numEntries = 0;
- return nullptr;
- }
- *numEntries = stat_infos.size();
- hdfsFileInfo *file_infos = new hdfsFileInfo[stat_infos.size()];
- for(std::vector<StatInfo>::size_type i = 0; i < stat_infos.size(); i++) {
- StatInfoToHdfsFileInfo(&file_infos[i], stat_infos.at(i));
- }
-
- return file_infos;
- } catch (const std::exception & e) {
- ReportException(e);
+ if (!CheckSystem(fs)) {
+ *numEntries = 0;
+ return nullptr;
+ }
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return nullptr;
+ }
+ std::vector<StatInfo> stat_infos;
+ Status stat = fs->get_impl()->GetListing(*abs_path, &stat_infos);
+ if (!stat.ok()) {
+ Error(stat);
*numEntries = 0;
return nullptr;
- } catch (...) {
- ReportCaughtNonException();
+ }
+ if(stat_infos.empty()){
*numEntries = 0;
return nullptr;
}
+ *numEntries = stat_infos.size();
+ hdfsFileInfo *file_infos = new hdfsFileInfo[stat_infos.size()];
+ for(std::vector<StatInfo>::size_type i = 0; i < stat_infos.size(); i++) {
+ StatInfoToHdfsFileInfo(&file_infos[i], stat_infos.at(i));
+ }
+
+ return file_infos;
+ } catch (const std::exception & e) {
+ ReportException(e);
+ *numEntries = 0;
+ return nullptr;
+ } catch (...) {
+ ReportCaughtNonException();
+ *numEntries = 0;
+ return nullptr;
+ }
}
+HDFS_API
void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries)
{
- errno = 0;
- int i;
- for (i = 0; i < numEntries; ++i) {
- delete[] hdfsFileInfo[i].mName;
- delete[] hdfsFileInfo[i].mOwner;
- delete[] hdfsFileInfo[i].mGroup;
- }
- delete[] hdfsFileInfo;
+ resetErrorForApiCall("hdfsFreeFileInfo");
+ int i;
+ for (i = 0; i < numEntries; ++i) {
+ delete[] hdfsFileInfo[i].mName;
+ delete[] hdfsFileInfo[i].mOwner;
+ delete[] hdfsFileInfo[i].mGroup;
+ }
+ delete[] hdfsFileInfo;
}
+HDFS_API
int hdfsCreateDirectory(hdfsFS fs, const char* path) {
+ resetErrorForApiCall("hdfsCreateDirectory");
try {
- errno = 0;
if (!CheckSystem(fs)) {
return -1;
}
@@ -891,45 +930,39 @@ int hdfsCreateDirectory(hdfsFS fs, const char* path) {
return -1;
}
Status stat;
- //Use default permissions and set true for creating all non-existant parent directories
+ //Use default permissions and set true for creating all non-existent parent directories
stat = fs->get_impl()->Mkdirs(*abs_path, FileSystem::GetDefaultPermissionMask(), true);
if (!stat.ok()) {
return Error(stat);
}
return 0;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
- }
+ } CATCH_AND_REPORT
}
+HDFS_API
int hdfsDelete(hdfsFS fs, const char* path, int recursive) {
+ resetErrorForApiCall("hdfsDelete");
try {
- errno = 0;
- if (!CheckSystem(fs)) {
- return -1;
- }
- const optional<std::string> abs_path = getAbsolutePath(fs, path);
- if(!abs_path) {
- return -1;
- }
- Status stat;
- stat = fs->get_impl()->Delete(*abs_path, recursive);
- if (!stat.ok()) {
- return Error(stat);
- }
- return 0;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
+ if (!CheckSystem(fs)) {
+ return -1;
+ }
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
+ }
+ Status stat;
+ stat = fs->get_impl()->Delete(*abs_path, recursive);
+ if (!stat.ok()) {
+ return Error(stat);
}
+ return 0;
+ } CATCH_AND_REPORT
}
+HDFS_API
int hdfsRename(hdfsFS fs, const char* oldPath, const char* newPath) {
+ resetErrorForApiCall("hdfsRename");
try {
- errno = 0;
if (!CheckSystem(fs)) {
return -1;
}
@@ -944,106 +977,99 @@ int hdfsRename(hdfsFS fs, const char* oldPath, const char* newPath) {
return Error(stat);
}
return 0;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
- }
+ } CATCH_AND_REPORT
}
+HDFS_API
int hdfsChmod(hdfsFS fs, const char* path, short mode){
+ resetErrorForApiCall("hdfsChmod");
try {
- errno = 0;
- if (!CheckSystem(fs)) {
- return -1;
- }
- const optional<std::string> abs_path = getAbsolutePath(fs, path);
- if(!abs_path) {
- return -1;
- }
- Status stat = FileSystem::CheckValidPermissionMask(mode);
- if (!stat.ok()) {
- return Error(stat);
- }
- stat = fs->get_impl()->SetPermission(*abs_path, mode);
- if (!stat.ok()) {
- return Error(stat);
- }
- return 0;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
+ if (!CheckSystem(fs)) {
+ return -1;
+ }
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
+ }
+ Status stat = FileSystem::CheckValidPermissionMask(mode);
+ if (!stat.ok()) {
+ return Error(stat);
+ }
+ stat = fs->get_impl()->SetPermission(*abs_path, mode);
+ if (!stat.ok()) {
+ return Error(stat);
}
+ return 0;
+ } CATCH_AND_REPORT
}
+HDFS_API
int hdfsChown(hdfsFS fs, const char* path, const char *owner, const char *group){
+ resetErrorForApiCall("hdfsChown");
try {
- errno = 0;
- if (!CheckSystem(fs)) {
- return -1;
- }
- const optional<std::string> abs_path = getAbsolutePath(fs, path);
- if(!abs_path) {
- return -1;
- }
- std::string own = (owner) ? owner : "";
- std::string grp = (group) ? group : "";
+ if (!CheckSystem(fs)) {
+ return -1;
+ }
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
+ }
+ std::string own = (owner) ? owner : "";
+ std::string grp = (group) ? group : "";
- Status stat;
- stat = fs->get_impl()->SetOwner(*abs_path, own, grp);
- if (!stat.ok()) {
- return Error(stat);
- }
- return 0;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
+ Status stat;
+ stat = fs->get_impl()->SetOwner(*abs_path, own, grp);
+ if (!stat.ok()) {
+ return Error(stat);
}
+ return 0;
+ } CATCH_AND_REPORT
}
-hdfsFileInfo * hdfsFind(hdfsFS fs, const char* path, const char* name, uint32_t * numEntries){
+HDFS_EXT_API
+hdfsFileInfo * hdfsFind(hdfsFS fs, const char* path, const char* name, uint32_t * numEntries)
+{
+ resetErrorForApiCall("hdfsFind");
try {
- errno = 0;
- if (!CheckSystem(fs)) {
- *numEntries = 0;
- return nullptr;
- }
-
- std::vector<StatInfo> stat_infos;
- Status stat = fs->get_impl()->Find(path, name, hdfs::FileSystem::GetDefaultFindMaxDepth(), &stat_infos);
- if (!stat.ok()) {
- Error(stat);
- *numEntries = 0;
- return nullptr;
- }
- //Existing API expects nullptr if size is 0
- if(stat_infos.empty()){
- *numEntries = 0;
- return nullptr;
- }
- *numEntries = stat_infos.size();
- hdfsFileInfo *file_infos = new hdfsFileInfo[stat_infos.size()];
- for(std::vector<StatInfo>::size_type i = 0; i < stat_infos.size(); i++) {
- StatInfoToHdfsFileInfo(&file_infos[i], stat_infos.at(i));
- }
+ if (!CheckSystem(fs)) {
+ *numEntries = 0;
+ return nullptr;
+ }
- return file_infos;
- } catch (const std::exception & e) {
- ReportException(e);
+ std::vector<StatInfo> stat_infos;
+ Status stat = fs->get_impl()->Find(path, name, hdfs::FileSystem::GetDefaultFindMaxDepth(), &stat_infos);
+ if (!stat.ok()) {
+ Error(stat);
*numEntries = 0;
return nullptr;
- } catch (...) {
- ReportCaughtNonException();
+ }
+ //Existing API expects nullptr if size is 0
+ if(stat_infos.empty()){
*numEntries = 0;
return nullptr;
}
+ *numEntries = stat_infos.size();
+ hdfsFileInfo *file_infos = new hdfsFileInfo[stat_infos.size()];
+ for(std::vector<StatInfo>::size_type i = 0; i < stat_infos.size(); i++) {
+ StatInfoToHdfsFileInfo(&file_infos[i], stat_infos.at(i));
+ }
+
+ return file_infos;
+ } catch (const std::exception & e) {
+ ReportException(e);
+ *numEntries = 0;
+ return nullptr;
+ } catch (...) {
+ ReportCaughtNonException();
+ *numEntries = 0;
+ return nullptr;
+ }
}
+HDFS_EXT_API
int hdfsCreateSnapshot(hdfsFS fs, const char* path, const char* name) {
+ resetErrorForApiCall("hdfsCreateSnapshot");
try {
- errno = 0;
if (!CheckSystem(fs)) {
return -1;
}
@@ -1061,16 +1087,13 @@ int hdfsCreateSnapshot(hdfsFS fs, const char* path, const char* name) {
return Error(stat);
}
return 0;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
- }
+ } CATCH_AND_REPORT
}
+HDFS_EXT_API
int hdfsDeleteSnapshot(hdfsFS fs, const char* path, const char* name) {
+ resetErrorForApiCall("hdfsDeleteSnapshot");
try {
- errno = 0;
if (!CheckSystem(fs)) {
return -1;
}
@@ -1087,16 +1110,13 @@ int hdfsDeleteSnapshot(hdfsFS fs, const char* path, const char* name) {
return Error(stat);
}
return 0;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
- }
+ } CATCH_AND_REPORT
}
+HDFS_EXT_API
int hdfsAllowSnapshot(hdfsFS fs, const char* path) {
+ resetErrorForApiCall("hdfsAllowSnapshot");
try {
- errno = 0;
if (!CheckSystem(fs)) {
return -1;
}
@@ -1110,16 +1130,13 @@ int hdfsAllowSnapshot(hdfsFS fs, const char* path) {
return Error(stat);
}
return 0;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
- }
+ } CATCH_AND_REPORT
}
+HDFS_EXT_API
int hdfsDisallowSnapshot(hdfsFS fs, const char* path) {
+ resetErrorForApiCall("hdfsDisallowSnapshot");
try {
- errno = 0;
if (!CheckSystem(fs)) {
return -1;
}
@@ -1133,18 +1150,15 @@ int hdfsDisallowSnapshot(hdfsFS fs, const char* path) {
return Error(stat);
}
return 0;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
- }
+ } CATCH_AND_REPORT
}
+HDFS_API
tSize hdfsPread(hdfsFS fs, hdfsFile file, tOffset position, void *buffer,
tSize length) {
+ resetErrorForApiCall("hdfsPread");
try
{
- errno = 0;
if (!CheckSystemAndHandle(fs, file)) {
return -1;
}
@@ -1155,17 +1169,14 @@ tSize hdfsPread(hdfsFS fs, hdfsFile file, tOffset position, void *buffer,
return Error(stat);
}
return (tSize)len;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
- }
+ } CATCH_AND_REPORT
}
+HDFS_API
tSize hdfsRead(hdfsFS fs, hdfsFile file, void *buffer, tSize length) {
+ resetErrorForApiCall("hdfsRead");
try
{
- errno = 0;
if (!CheckSystemAndHandle(fs, file)) {
return -1;
}
@@ -1177,67 +1188,63 @@ tSize hdfsRead(hdfsFS fs, hdfsFile file, void *buffer, tSize length) {
}
return (tSize)len;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
- }
+ } CATCH_AND_REPORT
}
+HDFS_API
int hdfsUnbufferFile(hdfsFile file) {
+ resetErrorForApiCall("hdfsUnbufferFile");
//Currently we are not doing any buffering
CheckHandle(file);
return -1;
}
+HDFS_API
int hdfsFileGetReadStatistics(hdfsFile file, struct hdfsReadStatistics **stats) {
+ resetErrorForApiCall("hdfsFileGetReadStatistics");
try
- {
- errno = 0;
- if (!CheckHandle(file)) {
- return -1;
- }
- *stats = new hdfsReadStatistics;
- memset(*stats, 0, sizeof(hdfsReadStatistics));
- (*stats)->totalBytesRead = file->get_impl()->get_bytes_read();
- return 0;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
+ {
+ if (!CheckHandle(file)) {
+ return -1;
}
+ *stats = new hdfsReadStatistics;
+ memset(*stats, 0, sizeof(hdfsReadStatistics));
+ (*stats)->totalBytesRead = file->get_impl()->get_bytes_read();
+ return 0;
+ } CATCH_AND_REPORT
}
+HDFS_API
int hdfsFileClearReadStatistics(hdfsFile file) {
+ resetErrorForApiCall("hdfsFileClearReadStatistics");
try
- {
- errno = 0;
- if (!CheckHandle(file)) {
- return -1;
- }
- file->get_impl()->clear_bytes_read();
- return 0;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
+ {
+ if (!CheckHandle(file)) {
+ return -1;
}
+ file->get_impl()->clear_bytes_read();
+ return 0;
+ } CATCH_AND_REPORT
}
+HDFS_API
int64_t hdfsReadStatisticsGetRemoteBytesRead(const struct hdfsReadStatistics *stats) {
- return stats->totalBytesRead - stats->totalLocalBytesRead;
+ resetErrorForApiCall("hdfsReadStatisticsGetRemoteBytesRead");
+ return stats->totalBytesRead - stats->totalLocalBytesRead;
}
+HDFS_API
void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats) {
- errno = 0;
- delete stats;
+ resetErrorForApiCall("hdfsFileFreeReadStatistics");
+ delete stats;
}
/* 0 on success, -1 on error*/
+HDFS_API
int hdfsSeek(hdfsFS fs, hdfsFile file, tOffset desiredPos) {
+ resetErrorForApiCall("hdfsSeek");
try
{
- errno = 0;
if (!CheckSystemAndHandle(fs, file)) {
return -1;
}
@@ -1249,17 +1256,14 @@ int hdfsSeek(hdfsFS fs, hdfsFile file, tOffset desiredPos) {
}
return 0;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
- }
+ } CATCH_AND_REPORT
}
+HDFS_API
tOffset hdfsTell(hdfsFS fs, hdfsFile file) {
+ resetErrorForApiCall("hdfsTell");
try
{
- errno = 0;
if (!CheckSystemAndHandle(fs, file)) {
return -1;
}
@@ -1271,36 +1275,30 @@ tOffset hdfsTell(hdfsFS fs, hdfsFile file) {
}
return offset;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
- }
+ } CATCH_AND_REPORT
}
/* extended API */
+HDFS_EXT_API
int hdfsCancel(hdfsFS fs, hdfsFile file) {
+ resetErrorForApiCall("hdfsCancel");
try
{
- errno = 0;
if (!CheckSystemAndHandle(fs, file)) {
return -1;
}
static_cast<FileHandleImpl*>(file->get_impl())->CancelOperations();
return 0;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
- }
+ } CATCH_AND_REPORT
}
+HDFS_EXT_API
int hdfsGetBlockLocations(hdfsFS fs, const char *path, struct hdfsBlockLocations ** locations_out)
{
+ resetErrorForApiCall("hdfsGetBlockLocations");
try
{
- errno = 0;
if (!CheckSystem(fs)) {
return -1;
}
@@ -1364,15 +1362,12 @@ int hdfsGetBlockLocations(hdfsFS fs, const char *path, struct hdfsBlockLocations
}
return 0;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
- }
+ } CATCH_AND_REPORT
}
+HDFS_EXT_API
int hdfsFreeBlockLocations(struct hdfsBlockLocations * blockLocations) {
- errno = 0;
+ resetErrorForApiCall("hdfsFreeBlockLocations");
if (blockLocations == nullptr)
return 0;
@@ -1391,48 +1386,50 @@ int hdfsFreeBlockLocations(struct hdfsBlockLocations * blockLocations) {
return 0;
}
+HDFS_API
char*** hdfsGetHosts(hdfsFS fs, const char* path, tOffset start, tOffset length) {
+ resetErrorForApiCall("hdfsGetHosts");
try
- {
- errno = 0;
- if (!CheckSystem(fs)) {
- return nullptr;
- }
- const optional<std::string> abs_path = getAbsolutePath(fs, path);
- if(!abs_path) {
- return nullptr;
- }
- std::shared_ptr<FileBlockLocation> ppLocations;
- Status stat = fs->get_impl()->GetBlockLocations(*abs_path, start, length, &ppLocations);
- if (!stat.ok()) {
- Error(stat);
- return nullptr;
- }
- const std::vector<BlockLocation> & ppBlockLocations = ppLocations->getBlockLocations();
- char ***hosts = new char**[ppBlockLocations.size() + 1];
- for (size_t i=0; i < ppBlockLocations.size(); i++) {
- const std::vector<DNInfo> & ppDNInfos = ppBlockLocations[i].getDataNodes();
- hosts[i] = new char*[ppDNInfos.size() + 1];
- for (size_t j=0; j < ppDNInfos.size(); j++) {
- auto ppDNInfo = ppDNInfos[j];
- hosts[i][j] = new char[ppDNInfo.getHostname().size() + 1];
- strncpy(hosts[i][j], ppDNInfo.getHostname().c_str(), ppDNInfo.getHostname().size() + 1);
- }
- hosts[i][ppDNInfos.size()] = nullptr;
- }
- hosts[ppBlockLocations.size()] = nullptr;
- return hosts;
- } catch (const std::exception & e) {
- ReportException(e);
+ {
+ if (!CheckSystem(fs)) {
+ return nullptr;
+ }
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
return nullptr;
- } catch (...) {
- ReportCaughtNonException();
+ }
+ std::shared_ptr<FileBlockLocation> ppLocations;
+ Status stat = fs->get_impl()->GetBlockLocations(*abs_path, start, length, &ppLocations);
+ if (!stat.ok()) {
+ Error(stat);
return nullptr;
}
+ const std::vector<BlockLocation> & ppBlockLocations = ppLocations->getBlockLocations();
+ char ***hosts = new char**[ppBlockLocations.size() + 1];
+ for (size_t i=0; i < ppBlockLocations.size(); i++) {
+ const std::vector<DNInfo> & ppDNInfos = ppBlockLocations[i].getDataNodes();
+ hosts[i] = new char*[ppDNInfos.size() + 1];
+ for (size_t j=0; j < ppDNInfos.size(); j++) {
+ auto ppDNInfo = ppDNInfos[j];
+ hosts[i][j] = new char[ppDNInfo.getHostname().size() + 1];
+ strncpy(hosts[i][j], ppDNInfo.getHostname().c_str(), ppDNInfo.getHostname().size() + 1);
+ }
+ hosts[i][ppDNInfos.size()] = nullptr;
+ }
+ hosts[ppBlockLocations.size()] = nullptr;
+ return hosts;
+ } catch (const std::exception & e) {
+ ReportException(e);
+ return nullptr;
+ } catch (...) {
+ ReportCaughtNonException();
+ return nullptr;
+ }
}
+HDFS_API
void hdfsFreeHosts(char ***blockHosts) {
- errno = 0;
+ resetErrorForApiCall("hdfsFreeHosts");
if (blockHosts == nullptr)
return;
@@ -1495,16 +1492,20 @@ event_response file_callback_glue(libhdfspp_file_event_callback handler,
return event_response::ok();
}
+HDFS_EXT_API
int hdfsPreAttachFSMonitor(libhdfspp_fs_event_callback handler, int64_t cookie)
{
+ resetErrorForApiCall("hdfsPreAttachFSMonitor");
fs_event_callback callback = std::bind(fs_callback_glue, handler, cookie, _1, _2, _3);
fsEventCallback = callback;
return 0;
}
+HDFS_EXT_API
int hdfsPreAttachFileMonitor(libhdfspp_file_event_callback handler, int64_t cookie)
{
+ resetErrorForApiCall("hdfsPreAttachFileMonitor");
file_event_callback callback = std::bind(file_callback_glue, handler, cookie, _1, _2, _3, _4);
fileEventCallback = callback;
return 0;
@@ -1541,11 +1542,12 @@ hdfsBuilder::hdfsBuilder(const char * directory) :
config = LoadDefault(loader);
}
+HDFS_API
struct hdfsBuilder *hdfsNewBuilder(void)
{
+ resetErrorForApiCall("hdfsNewBuilder");
try
{
- errno = 0;
return new struct hdfsBuilder();
} catch (const std::exception & e) {
ReportException(e);
@@ -1556,37 +1558,42 @@ struct hdfsBuilder *hdfsNewBuilder(void)
}
}
+HDFS_API
void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn)
{
- errno = 0;
+ resetErrorForApiCall("hdfsBuilderSetNameNode");
bld->overrideHost = std::string(nn);
}
+HDFS_API
void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port)
{
- errno = 0;
+ resetErrorForApiCall("hdfsBuilderSetNameNodePort");
bld->overridePort = port;
}
+HDFS_API
void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName)
{
- errno = 0;
+ resetErrorForApiCall("hdfsBuilderSetUserName");
if (userName && *userName) {
bld->user = std::string(userName);
}
}
+HDFS_API
void hdfsBuilderSetForceNewInstance(struct hdfsBuilder *bld) {
+ resetErrorForApiCall("hdfsBuilderSetForceNewInstance");
//libhdfspp always returns a new instance, so nothing to do
(void)bld;
- errno = 0;
}
+HDFS_API
void hdfsFreeBuilder(struct hdfsBuilder *bld)
{
+ resetErrorForApiCall("hdfsFreeBuilder");
try
{
- errno = 0;
delete bld;
} catch (const std::exception & e) {
ReportException(e);
@@ -1595,12 +1602,12 @@ void hdfsFreeBuilder(struct hdfsBuilder *bld)
}
}
-int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key,
- const char *val)
+HDFS_API
+int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key, const char *val)
{
+ resetErrorForApiCall("hdfsBuilderConfSetStr");
try
{
- errno = 0;
optional<HdfsConfiguration> newConfig = bld->loader.OverlayValue(bld->config, key, val);
if (newConfig)
{
@@ -1612,6 +1619,17 @@ int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key,
ReportError(EINVAL, "Could not change Builder value");
return -1;
}
+ } CATCH_AND_REPORT
+}
+
+HDFS_API
+int hdfsConfGetStr(const char *key, char **val)
+{
+ resetErrorForApiCall("hdfsConfGetStr");
+ try
+ {
+ hdfsBuilder builder;
+ return hdfsBuilderConfGetStr(&builder, key, val);
} catch (const std::exception & e) {
return ReportException(e);
} catch (...) {
@@ -1619,52 +1637,39 @@ int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key,
}
}
+HDFS_API
void hdfsConfStrFree(char *val)
{
- errno = 0;
+ resetErrorForApiCall("hdfsConfStrFree");
free(val);
}
+HDFS_API
hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) {
+ resetErrorForApiCall("hdfsBuilderConnect");
return doHdfsConnect(bld->overrideHost, bld->overridePort, bld->user, bld->config.GetOptions());
}
-int hdfsConfGetStr(const char *key, char **val)
-{
- try
- {
- errno = 0;
- hdfsBuilder builder;
- return hdfsBuilderConfGetStr(&builder, key, val);
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
- }
-}
-
+HDFS_API
int hdfsConfGetInt(const char *key, int32_t *val)
{
+ resetErrorForApiCall("hdfsConfGetInt");
try
{
- errno = 0;
hdfsBuilder builder;
return hdfsBuilderConfGetInt(&builder, key, val);
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
- }
+ } CATCH_AND_REPORT
}
//
// Extended builder interface
//
+HDFS_EXT_API
struct hdfsBuilder *hdfsNewBuilderFromDirectory(const char * configDirectory)
{
+ resetErrorForApiCall("hdfsNewBuilderFromDirectory");
try
{
- errno = 0;
return new struct hdfsBuilder(configDirectory);
} catch (const std::exception & e) {
ReportException(e);
@@ -1675,12 +1680,12 @@ struct hdfsBuilder *hdfsNewBuilderFromDirectory(const char * configDirectory)
}
}
-int hdfsBuilderConfGetStr(struct hdfsBuilder *bld, const char *key,
- char **val)
+HDFS_EXT_API
+int hdfsBuilderConfGetStr(struct hdfsBuilder *bld, const char *key, char **val)
{
+ resetErrorForApiCall("hdfsBuilderConfGetStr");
try
{
- errno = 0;
optional<std::string> value = bld->config.Get(key);
if (value)
{
@@ -1693,11 +1698,7 @@ int hdfsBuilderConfGetStr(struct hdfsBuilder *bld, const char *key,
*val = nullptr;
}
return 0;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
- }
+ } CATCH_AND_REPORT
}
// If we're running on a 32-bit platform, we might get 64-bit values that
@@ -1708,11 +1709,12 @@ bool isValidInt(int64_t value)
value <= std::numeric_limits<int>::max());
}
+HDFS_EXT_API
int hdfsBuilderConfGetInt(struct hdfsBuilder *bld, const char *key, int32_t *val)
{
+ resetErrorForApiCall("hdfsBuilderConfGetInt");
try
{
- errno = 0;
// Pull from default configuration
optional<int64_t> value = bld->config.GetInt(key);
if (value)
@@ -1727,18 +1729,15 @@ int hdfsBuilderConfGetInt(struct hdfsBuilder *bld, const char *key, int32_t *val
// If not found, don't change val
ReportError(EINVAL, "Could not get Builder value");
return 0;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
- }
+ } CATCH_AND_REPORT
}
+HDFS_EXT_API
int hdfsBuilderConfGetLong(struct hdfsBuilder *bld, const char *key, int64_t *val)
{
+ resetErrorForApiCall("hdfsBuilderConfGetLong");
try
{
- errno = 0;
// Pull from default configuration
optional<int64_t> value = bld->config.GetInt(key);
if (value)
@@ -1749,11 +1748,7 @@ int hdfsBuilderConfGetLong(struct hdfsBuilder *bld, const char *key, int64_t *va
// If not found, don't change val
ReportError(EINVAL, "Could not get Builder value");
return 0;
- } catch (const std::exception & e) {
- return ReportException(e);
- } catch (...) {
- return ReportCaughtNonException();
- }
+ } CATCH_AND_REPORT
}
/**
@@ -1829,15 +1824,21 @@ void CForwardingLogger::FreeLogData(LogData *data) {
}
+HDFS_EXT_API
LogData *hdfsCopyLogData(LogData *data) {
+ resetErrorForApiCall("hdfsCopyLogData");
return CForwardingLogger::CopyLogData(data);
}
+HDFS_EXT_API
void hdfsFreeLogData(LogData *data) {
+ resetErrorForApiCall("hdfsFreeLogData");
CForwardingLogger::FreeLogData(data);
}
+HDFS_EXT_API
void hdfsSetLogFunction(void (*callback)(LogData*)) {
+ resetErrorForApiCall("hdfsSetLogFunction");
CForwardingLogger *logger = new CForwardingLogger();
logger->SetCallback(callback);
LogManager::SetLoggerImplementation(std::unique_ptr<LoggerInterface>(logger));
@@ -1849,8 +1850,8 @@ static bool IsLevelValid(int component) {
return true;
}
-
// should use __builtin_popcnt as optimization on some platforms
+// *but maybe not Intel: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=62011
static int popcnt(int val) {
int bits = sizeof(val) * 8;
int count = 0;
@@ -1869,24 +1870,27 @@ static bool IsComponentValid(int component) {
return true;
}
+HDFS_EXT_API
int hdfsEnableLoggingForComponent(int component) {
- errno = 0;
+ resetErrorForApiCall("hdfsEnableLoggingForComponent");
if(!IsComponentValid(component))
return -1;
LogManager::EnableLogForComponent(static_cast<LogSourceComponent>(component));
return 0;
}
+HDFS_EXT_API
int hdfsDisableLoggingForComponent(int component) {
- errno = 0;
+ resetErrorForApiCall("hdfsDisableLoggingForComponent");
if(!IsComponentValid(component))
return -1;
LogManager::DisableLogForComponent(static_cast<LogSourceComponent>(component));
return 0;
}
+HDFS_EXT_API
int hdfsSetLoggingLevel(int level) {
- errno = 0;
+ resetErrorForApiCall("hdfsSetLoggingLevel");
if(!IsLevelValid(level))
return -1;
LogManager::SetLogLevel(static_cast<LogLevel>(level));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/52] [abbrv] hadoop git commit: HDFS-11767: libhdfs++: SASL events
should be scoped closer to usage. Contributed by James Clampffer
Posted by jh...@apache.org.
HDFS-11767: libhdfs++: SASL events should be scoped closer to usage. Contributed by James Clampffer
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/57cdad73
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/57cdad73
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/57cdad73
Branch: refs/heads/HDFS-8707
Commit: 57cdad73deecdb39fbe3c43688dbd71bcfd0e31a
Parents: b584e34
Author: James <jh...@apache.org>
Authored: Thu May 11 12:34:35 2017 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc | 7 ++++---
.../src/main/native/libhdfspp/lib/rpc/sasl_protocol.h | 1 -
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57cdad73/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
index 83d4f88..49abe58 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
@@ -65,14 +65,12 @@ SaslProtocol::SaslProtocol(const std::string & cluster_name,
SaslProtocol::~SaslProtocol()
{
- std::lock_guard<std::mutex> state_lock(sasl_state_lock_);
- event_handlers_->call("SASL End", cluster_name_.c_str(), 0);
+ assert(state_ != kNegotiate);
}
void SaslProtocol::SetEventHandlers(std::shared_ptr<LibhdfsEvents> event_handlers) {
std::lock_guard<std::mutex> state_lock(sasl_state_lock_);
event_handlers_ = event_handlers;
- event_handlers_->call("SASL Start", cluster_name_.c_str(), 0);
} // SetEventHandlers() method
void SaslProtocol::Authenticate(std::function<void(const Status & status, const AuthInfo new_auth_info)> callback)
@@ -81,6 +79,7 @@ void SaslProtocol::Authenticate(std::function<void(const Status & status, const
callback_ = callback;
state_ = kNegotiate;
+ event_handlers_->call("SASL Start", cluster_name_.c_str(), 0);
std::shared_ptr<RpcSaslProto> req_msg = std::make_shared<RpcSaslProto>();
req_msg->set_state(RpcSaslProto_SaslState_NEGOTIATE);
@@ -353,6 +352,8 @@ bool SaslProtocol::SendSaslMessage(RpcSaslProto & message)
bool SaslProtocol::AuthComplete(const Status & status, const AuthInfo & auth_info)
{
assert(lock_held(sasl_state_lock_)); // Must be holding lock before calling
+ state_ = kComplete;
+ event_handlers_->call("SASL End", cluster_name_.c_str(), 0);
// RpcConnection might have been freed when we weren't looking. Lock it
// to make sure it's there long enough for us
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57cdad73/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.h
index 3816cf9..a46ae08 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.h
@@ -56,7 +56,6 @@ private:
enum State {
kUnstarted,
kNegotiate,
- kAuthenticate,
kComplete
};
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/52] [abbrv] hadoop git commit: HDFS-12103: libhdfs++: Provide
workaround to support cancel on filesystem connect until HDFS-11437 is
resolved. Contributed by James Clampffer.
Posted by jh...@apache.org.
HDFS-12103: libhdfs++: Provide workaround to support cancel on filesystem connect until HDFS-11437 is resolved. Contributed by James Clampffer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3db261d4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3db261d4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3db261d4
Branch: refs/heads/HDFS-8707
Commit: 3db261d45bab46759f4049675483633822b1d090
Parents: 23afd50
Author: James Clampffer <ja...@hp.com>
Authored: Mon Jul 10 14:23:13 2017 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../src/main/native/libhdfspp/lib/fs/filesystem.cc | 10 ++++++----
.../src/main/native/libhdfspp/lib/rpc/rpc_engine.cc | 4 +++-
2 files changed, 9 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3db261d4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
index 9ef3aa0..761ff86 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
@@ -288,12 +288,14 @@ int FileSystemImpl::WorkerThreadCount() {
}
bool FileSystemImpl::CancelPendingConnect() {
- if(!connect_callback_.IsCallbackSet()) {
- LOG_DEBUG(kFileSystem, << "FileSystemImpl@" << this << "::CancelPendingConnect called before Connect started");
- return false;
- }
if(connect_callback_.IsCallbackAccessed()) {
+ // Temp fix for failover hangs, allow CancelPendingConnect to be called so it can push a flag through the RPC engine
LOG_DEBUG(kFileSystem, << "FileSystemImpl@" << this << "::CancelPendingConnect called after Connect completed");
+ return nn_.CancelPendingConnect();
+ }
+
+ if(!connect_callback_.IsCallbackSet()) {
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl@" << this << "::CancelPendingConnect called before Connect started");
return false;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3db261d4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
index 98c41da..ba55565 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
@@ -220,7 +220,9 @@ void RpcEngine::RpcCommsError(
RetryAction retry = RetryAction::fail(""); // Default to fail
- if (status.notWorthRetry()) {
+ if(connect_canceled_) {
+ retry = RetryAction::fail("Operation canceled");
+ } else if (status.notWorthRetry()) {
retry = RetryAction::fail(status.ToString().c_str());
} else if (retry_policy()) {
retry = retry_policy()->ShouldRetry(status, req->IncrementRetryCount(), req->get_failover_count(), true);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/52] [abbrv] hadoop git commit: HDFS-11014: libhdfs++: Make
connection to HA clusters faster. Contributed by James Clampffer.
Posted by jh...@apache.org.
HDFS-11014: libhdfs++: Make connection to HA clusters faster. Contributed by James Clampffer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/59a39269
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/59a39269
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/59a39269
Branch: refs/heads/HDFS-8707
Commit: 59a39269463bc6fd76b1e5e30cc8ccde5250e7fb
Parents: 6dd47ca
Author: James <jh...@apache.org>
Authored: Mon Oct 17 11:57:45 2016 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../src/main/native/libhdfspp/include/hdfspp/options.h | 2 +-
.../src/main/native/libhdfspp/lib/common/retry_policy.cc | 7 ++++++-
.../src/main/native/libhdfspp/lib/rpc/rpc_engine.cc | 2 +-
3 files changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59a39269/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
index 55093d0..b73a729 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
@@ -96,7 +96,7 @@ struct Options {
* Client failover attempts before failover gives up
**/
int failover_max_retries;
- static const unsigned int kDefaultFailoverMaxRetries = 15;
+ static const unsigned int kDefaultFailoverMaxRetries = 4;
/**
* Client failover attempts before failover gives up if server
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59a39269/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.cc
index ef78aca..148c674 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.cc
@@ -60,7 +60,12 @@ RetryAction FixedDelayWithFailover::ShouldRetry(const Status &s, uint64_t retrie
{
// Try connecting to another NN in case this one keeps timing out
// Can add the backoff wait specified by dfs.client.failover.sleep.base.millis here
- return RetryAction::failover(delay_);
+ if(failovers == 0) {
+ // No delay on first failover if it looks like the NN was bad.
+ return RetryAction::failover(0);
+ } else {
+ return RetryAction::failover(delay_);
+ }
}
if(retries < max_retries_ && failovers < max_failover_retries_) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59a39269/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
index 4de6704..72a0c55 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
@@ -365,7 +365,7 @@ void RpcEngine::RpcCommsError(
ResolvedNamenodeInfo new_active_nn_info =
ha_persisted_info_->GetFailoverAndUpdate(last_endpoints_[0]/*reverse lookup*/);
- LOG_INFO(kRPC, << "Going to try connecting to alternate Datanode: " << new_active_nn_info.uri.str());
+ LOG_INFO(kRPC, << "Going to try connecting to alternate Namenode: " << new_active_nn_info.uri.str());
if(ha_persisted_info_->is_resolved()) {
last_endpoints_ = new_active_nn_info.endpoints;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/52] [abbrv] hadoop git commit: HDFS-11730: libhdfs++:
RpcConnection should handle authorization error call id. Contributed by James
Clampffer
Posted by jh...@apache.org.
HDFS-11730: libhdfs++: RpcConnection should handle authorization error call id. Contributed by James Clampffer
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b584e34f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b584e34f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b584e34f
Branch: refs/heads/HDFS-8707
Commit: b584e34f2f626c72b1a632bfc82da1247ff438d8
Parents: fdb88eb
Author: James <jh...@apache.org>
Authored: Wed May 3 12:03:29 2017 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../src/main/native/libhdfspp/include/hdfspp/status.h | 2 ++
.../src/main/native/libhdfspp/lib/common/status.cc | 13 +++++++++++++
.../native/libhdfspp/lib/rpc/rpc_connection_impl.cc | 4 +++-
3 files changed, 18 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b584e34f/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
index d2c32b2..6fc00b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
@@ -44,6 +44,8 @@ class Status {
static Status Error(const char *error_message);
static Status AuthenticationFailed();
static Status AuthenticationFailed(const char *msg);
+ static Status AuthorizationFailed();
+ static Status AuthorizationFailed(const char *msg);
static Status Canceled();
static Status PathNotFound(const char *msg);
static Status InvalidOffset(const char *msg);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b584e34f/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
index 590a036..5903553 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
@@ -143,6 +143,19 @@ Status Status::AuthenticationFailed(const char *msg) {
return Status(kAuthenticationFailed, formatted.c_str());
}
+Status Status::AuthorizationFailed() {
+ return Status::AuthorizationFailed(nullptr);
+}
+
+Status Status::AuthorizationFailed(const char *msg) {
+ std::string formatted = "AuthorizationFailed";
+ if(msg) {
+ formatted += ": ";
+ formatted += msg;
+ }
+ return Status(kPermissionDenied, formatted.c_str());
+}
+
Status Status::Canceled() {
return Status(kOperationCanceled, "Operation canceled");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b584e34f/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
index 7accaf8..1012a37 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
@@ -175,9 +175,11 @@ Status RpcConnection::HandleRpcResponse(std::shared_ptr<Response> response) {
auto req = RemoveFromRunningQueue(h.callid());
if (!req) {
- LOG_WARN(kRPC, << "RPC response with Unknown call id " << h.callid());
+ LOG_WARN(kRPC, << "RPC response with Unknown call id " << (int32_t)h.callid());
if((int32_t)h.callid() == RpcEngine::kCallIdSasl) {
return Status::AuthenticationFailed("You have an unsecured client connecting to a secured server");
+ } else if((int32_t)h.callid() == RpcEngine::kCallIdAuthorizationFailed) {
+ return Status::AuthorizationFailed("RPC call id indicates an authorization failure");
} else {
return Status::Error("Rpc response with unknown call id");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/52] [abbrv] hadoop git commit: HDFS-10705: libhdfs++: FileSystem
should have a convenience no-args ctor. Contributed by James Clampffer.
Posted by jh...@apache.org.
HDFS-10705: libhdfs++: FileSystem should have a convenience no-args ctor. Contributed by James Clampffer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9d1c9021
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9d1c9021
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9d1c9021
Branch: refs/heads/HDFS-8707
Commit: 9d1c9021591484145d14e889093fb773cc4b0557
Parents: 4f6cb5d
Author: James <jh...@apache.org>
Authored: Wed Sep 7 10:52:29 2016 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../main/native/libhdfspp/include/hdfspp/hdfspp.h | 17 ++++++++++++-----
.../src/main/native/libhdfspp/lib/fs/filesystem.cc | 15 ++++++++++++++-
2 files changed, 26 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d1c9021/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
index 4b88fe5..78ad594 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
@@ -155,18 +155,25 @@ class FileSystem {
* initializes the RPC connections to the NameNode and returns an
* FileSystem object.
*
+ * Note: The FileSystem takes ownership of the IoService passed in the
+ * constructor. The FileSystem destructor will call delete on it.
+ *
* If user_name is blank, the current user will be used for a default.
**/
- static FileSystem * New(
+ static FileSystem *New(
IoService *&io_service, const std::string &user_name, const Options &options);
- virtual void Connect(const std::string &server,
- const std::string &service,
+ /**
+ * Returns a new instance with default user and option, with the default IOService.
+ **/
+ static FileSystem *New();
+
+
+ virtual void Connect(const std::string &server, const std::string &service,
const std::function<void(const Status &, FileSystem *)> &handler) = 0;
/* Synchronous call of Connect */
- virtual Status Connect(const std::string &server,
- const std::string &service) = 0;
+ virtual Status Connect(const std::string &server, const std::string &service) = 0;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d1c9021/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
index d75939f..aa6f4ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
@@ -40,6 +40,9 @@ using ::asio::ip::tcp;
static constexpr uint16_t kDefaultPort = 8020;
+// forward declarations
+const std::string get_effective_user_name(const std::string &);
+
uint32_t FileSystem::GetDefaultFindMaxDepth() {
return std::numeric_limits<uint32_t>::max();
}
@@ -72,11 +75,21 @@ Status FileSystem::CheckValidReplication(uint16_t replication) {
* FILESYSTEM BASE CLASS
****************************************************************************/
-FileSystem * FileSystem::New(
+FileSystem *FileSystem::New(
IoService *&io_service, const std::string &user_name, const Options &options) {
return new FileSystemImpl(io_service, user_name, options);
}
+FileSystem *FileSystem::New() {
+ // No, this pointer won't be leaked. The FileSystem takes ownership.
+ IoService *io_service = IoService::New();
+ if(!io_service)
+ return nullptr;
+ std::string user_name = get_effective_user_name("");
+ Options options;
+ return new FileSystemImpl(io_service, user_name, options);
+}
+
/*****************************************************************************
* FILESYSTEM IMPLEMENTATION
****************************************************************************/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/52] [abbrv] hadoop git commit: HDFS-10785: libhdfs++: Implement
the rest of the tools. Contributed by Anatoli Schein
Posted by jh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/content_summary.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/content_summary.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/content_summary.cc
new file mode 100644
index 0000000..0dca36a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/content_summary.cc
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <hdfspp/content_summary.h>
+#include <sstream>
+#include <iomanip>
+
+namespace hdfs {
+
+ContentSummary::ContentSummary()
+: length(0),
+ filecount(0),
+ directorycount(0),
+ quota(0),
+ spaceconsumed(0),
+ spacequota(0) {
+}
+
+std::string ContentSummary::str(bool include_quota) const {
+ std::stringstream ss;
+ if(include_quota){
+ ss << this->quota << " "
+ << spacequota << " "
+ << spaceconsumed << " ";
+ }
+ ss << directorycount << " "
+ << filecount << " "
+ << length << " "
+ << path;
+ return ss.str();
+}
+
+std::string ContentSummary::str_du() const {
+ std::stringstream ss;
+ ss << std::left << std::setw(10) << length
+ << path;
+ return ss.str();
+}
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/fsinfo.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/fsinfo.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/fsinfo.cc
new file mode 100644
index 0000000..9f350a8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/fsinfo.cc
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <hdfspp/fsinfo.h>
+#include <sstream>
+#include <iomanip>
+
+namespace hdfs {
+
+FsInfo::FsInfo()
+ : capacity(0),
+ used(0),
+ remaining(0),
+ under_replicated(0),
+ corrupt_blocks(0),
+ missing_blocks(0),
+ missing_repl_one_blocks(0),
+ blocks_in_future(0) {
+}
+
+std::string FsInfo::str(const std::string fs_name) const {
+ std::string fs_name_label = "Filesystem";
+ std::string size = std::to_string(capacity);
+ std::string size_label = "Size";
+ std::string used = std::to_string(this->used);
+ std::string used_label = "Used";
+ std::string available = std::to_string(remaining);
+ std::string available_label = "Available";
+ std::string use_percentage = std::to_string(this->used * 100 / capacity) + "%";
+ std::string use_percentage_label = "Use%";
+ std::stringstream ss;
+ ss << std::left << std::setw(std::max(fs_name.size(), fs_name_label.size())) << fs_name_label
+ << std::right << std::setw(std::max(size.size(), size_label.size()) + 2) << size_label
+ << std::right << std::setw(std::max(used.size(), used_label.size()) + 2) << used_label
+ << std::right << std::setw(std::max(available.size(), available_label.size()) + 2) << available_label
+ << std::right << std::setw(std::max(use_percentage.size(), use_percentage_label.size()) + 2) << use_percentage_label
+ << std::endl
+ << std::left << std::setw(std::max(fs_name.size(), fs_name_label.size())) << fs_name
+ << std::right << std::setw(std::max(size.size(), size_label.size()) + 2) << size
+ << std::right << std::setw(std::max(used.size(), used_label.size()) + 2) << used
+ << std::right << std::setw(std::max(available.size(), available_label.size()) + 2) << available
+ << std::right << std::setw(std::max(use_percentage.size(), use_percentage_label.size()) + 2) << use_percentage;
+ return ss.str();
+}
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/statinfo.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/statinfo.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/statinfo.cc
new file mode 100644
index 0000000..2fb744f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/statinfo.cc
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <hdfspp/statinfo.h>
+#include <sys/stat.h>
+#include <sstream>
+#include <iomanip>
+
+namespace hdfs {
+
+StatInfo::StatInfo()
+ : file_type(0),
+ length(0),
+ permissions(0),
+ modification_time(0),
+ access_time(0),
+ block_replication(0),
+ blocksize(0),
+ fileid(0),
+ children_num(0) {
+}
+
+std::string StatInfo::str() const {
+ char perms[11];
+ perms[0] = file_type == StatInfo::IS_DIR ? 'd' : '-';
+ perms[1] = permissions & S_IRUSR? 'r' : '-';
+ perms[2] = permissions & S_IWUSR? 'w': '-';
+ perms[3] = permissions & S_IXUSR? 'x': '-';
+ perms[4] = permissions & S_IRGRP? 'r' : '-';
+ perms[5] = permissions & S_IWGRP? 'w': '-';
+ perms[6] = permissions & S_IXGRP? 'x': '-';
+ perms[7] = permissions & S_IROTH? 'r' : '-';
+ perms[8] = permissions & S_IWOTH? 'w': '-';
+ perms[9] = permissions & S_IXOTH? 'x': '-';
+ perms[10] = 0;
+
+ //Convert to seconds from milliseconds
+ const int time_field_length = 17;
+ time_t rawtime = modification_time/1000;
+ struct tm * timeinfo;
+ char buffer[time_field_length];
+ timeinfo = localtime(&rawtime);
+
+ strftime(buffer,time_field_length,"%Y-%m-%d %H:%M",timeinfo);
+ buffer[time_field_length-1] = 0; //null terminator
+ std::string time(buffer);
+
+ std::stringstream ss;
+ ss << std::left << std::setw(12) << perms
+ << std::left << std::setw(3) << (!block_replication ? "-" : std::to_string(block_replication))
+ << std::left << std::setw(15) << owner
+ << std::left << std::setw(15) << group
+ << std::right << std::setw(5) << length
+ << std::right << std::setw(time_field_length + 2) << time//modification_time
+ << " " << full_path;
+ return ss.str();
+}
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
index b46102a..9ef3aa0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
@@ -479,6 +479,15 @@ void FileSystemImpl::GetFileInfo(
nn_.GetFileInfo(path, handler);
}
+void FileSystemImpl::GetContentSummary(
+ const std::string &path,
+ const std::function<void(const Status &, const ContentSummary &)> &handler) {
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl::GetContentSummary("
+ << FMT_THIS_ADDR << ", path="
+ << path << ") called");
+
+ nn_.GetContentSummary(path, handler);
+}
void FileSystemImpl::GetFsStats(
const std::function<void(const Status &, const FsInfo &)> &handler) {
@@ -515,13 +524,16 @@ void FileSystemImpl::GetListing(
LOG_DEBUG(kFileSystem, << "FileSystemImpl::GetListing("
<< FMT_THIS_ADDR << ", path="
<< path << ") called");
-
+ std::string path_fixed = path;
+ if(path.back() != '/'){
+ path_fixed += "/";
+ }
// Caputure the state and push it into the shim
- auto callback = [this, path, handler](const Status &stat, const std::vector<StatInfo> & stat_infos, bool has_more) {
- GetListingShim(stat, stat_infos, has_more, path, handler);
+ auto callback = [this, path_fixed, handler](const Status &stat, const std::vector<StatInfo> & stat_infos, bool has_more) {
+ GetListingShim(stat, stat_infos, has_more, path_fixed, handler);
};
- nn_.GetListing(path, callback);
+ nn_.GetListing(path_fixed, callback);
}
@@ -772,6 +784,28 @@ void FileSystemImpl::DeleteSnapshot(const std::string &path,
nn_.DeleteSnapshot(path, name, handler);
}
+void FileSystemImpl::RenameSnapshot(const std::string &path,
+ const std::string &old_name, const std::string &new_name,
+ const std::function<void(const Status &)> &handler) {
+ LOG_DEBUG(kFileSystem,
+ << "FileSystemImpl::RenameSnapshot(" << FMT_THIS_ADDR << ", path=" << path <<
+ ", old_name=" << old_name << ", new_name=" << new_name << ") called");
+
+ if (path.empty()) {
+ handler(Status::InvalidArgument("RenameSnapshot: argument 'path' cannot be empty"));
+ return;
+ }
+ if (old_name.empty()) {
+ handler(Status::InvalidArgument("RenameSnapshot: argument 'old_name' cannot be empty"));
+ return;
+ }
+ if (new_name.empty()) {
+ handler(Status::InvalidArgument("RenameSnapshot: argument 'new_name' cannot be empty"));
+ return;
+ }
+
+ nn_.RenameSnapshot(path, old_name, new_name, handler);
+}
void FileSystemImpl::AllowSnapshot(const std::string &path,
const std::function<void(const Status &)> &handler) {
@@ -817,4 +851,8 @@ Options FileSystemImpl::get_options() {
return options_;
}
+std::string FileSystemImpl::get_cluster_name() {
+ return cluster_name_;
+}
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
index fbc3967..d97eb25 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
@@ -87,6 +87,10 @@ public:
Status GetFileInfo(const std::string &path, StatInfo & stat_info) override;
+ void GetContentSummary(const std::string &path,
+ const std::function<void(const Status &, const ContentSummary &)> &handler) override;
+ Status GetContentSummary(const std::string &path, ContentSummary & stat_info) override;
+
/**
* Retrieves the file system information such as the total raw size of all files in the filesystem
* and the raw capacity of the filesystem
@@ -160,6 +164,18 @@ public:
Status DeleteSnapshot(const std::string &path, const std::string &name) override;
/**
+ * Renames the directory snapshot specified by path from old_name to new_name
+ *
+ * @param path Path to the snapshotted directory (must be non-blank)
+ * @param old_name Current name of the snapshot (must be non-blank)
+ * @param new_name New name of the snapshot (must be non-blank)
+ **/
+ void RenameSnapshot(const std::string &path, const std::string &old_name,
+ const std::string &new_name, const std::function<void(const Status &)> &handler) override;
+ Status RenameSnapshot(const std::string &path, const std::string &old_name,
+ const std::string &new_name) override;
+
+ /**
* Allows snapshots to be made on the specified directory
*
* @param path Path to the directory to be made snapshottable (must be non-empty)
@@ -189,6 +205,8 @@ public:
Options get_options();
+ std::string get_cluster_name();
+
private:
/**
* The IoService must be the first member variable to ensure that it gets
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem_sync.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem_sync.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem_sync.cc
index 73be538..53c9e26 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem_sync.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem_sync.cc
@@ -252,6 +252,35 @@ Status FileSystemImpl::GetFileInfo(const std::string &path,
return stat;
}
+Status FileSystemImpl::GetContentSummary(const std::string &path,
+ ContentSummary & content_summary) {
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl::[sync]GetContentSummary("
+ << FMT_THIS_ADDR << ", path="
+ << path << ") called");
+
+ auto callstate = std::make_shared<std::promise<std::tuple<Status, ContentSummary>>>();
+ std::future<std::tuple<Status, ContentSummary>> future(callstate->get_future());
+
+ /* wrap async FileSystem::GetContentSummary with promise to make it a blocking call */
+ auto h = [callstate](const Status &s, const ContentSummary &si) {
+ callstate->set_value(std::make_tuple(s, si));
+ };
+
+ GetContentSummary(path, h);
+
+ /* block until promise is set */
+ auto returnstate = future.get();
+ Status stat = std::get<0>(returnstate);
+ ContentSummary cs = std::get<1>(returnstate);
+
+ if (!stat.ok()) {
+ return stat;
+ }
+
+ content_summary = cs;
+ return stat;
+}
+
Status FileSystemImpl::GetFsStats(FsInfo & fs_info) {
LOG_DEBUG(kFileSystem,
<< "FileSystemImpl::[sync]GetFsStats(" << FMT_THIS_ADDR << ") called");
@@ -510,6 +539,29 @@ Status FileSystemImpl::DeleteSnapshot(const std::string &path,
return stat;
}
+Status FileSystemImpl::RenameSnapshot(const std::string &path,
+ const std::string &old_name, const std::string &new_name) {
+ LOG_DEBUG(kFileSystem,
+ << "FileSystemImpl::[sync]RenameSnapshot(" << FMT_THIS_ADDR << ", path=" << path <<
+ ", old_name=" << old_name << ", new_name=" << new_name << ") called");
+
+ auto callstate = std::make_shared<std::promise<Status>>();
+ std::future<Status> future(callstate->get_future());
+
+ /* wrap async FileSystem::RenameSnapshot with promise to make it a blocking call */
+ auto h = [callstate](const Status &s) {
+ callstate->set_value(s);
+ };
+
+ RenameSnapshot(path, old_name, new_name, h);
+
+ /* block until promise is set */
+ auto returnstate = future.get();
+ Status stat = returnstate;
+
+ return stat;
+}
+
Status FileSystemImpl::AllowSnapshot(const std::string &path) {
LOG_DEBUG(kFileSystem,
<< "FileSystemImpl::[sync]AllowSnapshot(" << FMT_THIS_ADDR << ", path=" << path << ") called");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
index 9e2d90a..b5a6564 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
@@ -247,6 +247,45 @@ void NameNodeOperations::GetFileInfo(const std::string & path,
});
}
+void NameNodeOperations::GetContentSummary(const std::string & path,
+ std::function<void(const Status &, const ContentSummary &)> handler)
+{
+ using ::hadoop::hdfs::GetContentSummaryRequestProto;
+ using ::hadoop::hdfs::GetContentSummaryResponseProto;
+
+ LOG_TRACE(kFileSystem, << "NameNodeOperations::GetContentSummary("
+ << FMT_THIS_ADDR << ", path=" << path << ") called");
+
+ if (path.empty()) {
+ handler(Status::InvalidArgument("GetContentSummary: argument 'path' cannot be empty"), ContentSummary());
+ return;
+ }
+
+ GetContentSummaryRequestProto req;
+ req.set_path(path);
+
+ auto resp = std::make_shared<GetContentSummaryResponseProto>();
+
+ namenode_.GetContentSummary(&req, resp, [resp, handler, path](const Status &stat) {
+ if (stat.ok()) {
+ // For non-existant files, the server will respond with an OK message but
+ // no summary in the protobuf.
+ if(resp -> has_summary()){
+ struct ContentSummary content_summary;
+ content_summary.path = path;
+ ContentSummaryProtoToContentSummary(content_summary, resp->summary());
+ handler(stat, content_summary);
+ } else {
+ std::string errormsg = "No such file or directory: " + path;
+ Status statNew = Status::PathNotFound(errormsg.c_str());
+ handler(statNew, ContentSummary());
+ }
+ } else {
+ handler(stat, ContentSummary());
+ }
+ });
+}
+
void NameNodeOperations::GetFsStats(
std::function<void(const Status &, const FsInfo &)> handler) {
using ::hadoop::hdfs::GetFsStatusRequestProto;
@@ -300,7 +339,10 @@ void NameNodeOperations::GetListing(
for (::hadoop::hdfs::HdfsFileStatusProto const& fs : resp->dirlist().partiallisting()) {
StatInfo si;
si.path = fs.path();
- si.full_path = path + fs.path() + "/";
+ si.full_path = path + fs.path();
+ if(si.full_path.back() != '/'){
+ si.full_path += "/";
+ }
HdfsFileStatusProtoToStatInfo(si, fs);
stat_infos.push_back(si);
}
@@ -554,6 +596,41 @@ void NameNodeOperations::DeleteSnapshot(const std::string & path,
});
}
+void NameNodeOperations::RenameSnapshot(const std::string & path, const std::string & old_name,
+ const std::string & new_name, std::function<void(const Status &)> handler) {
+ using ::hadoop::hdfs::RenameSnapshotRequestProto;
+ using ::hadoop::hdfs::RenameSnapshotResponseProto;
+
+ LOG_TRACE(kFileSystem,
+ << "NameNodeOperations::RenameSnapshot(" << FMT_THIS_ADDR << ", path=" << path <<
+ ", old_name=" << old_name << ", new_name=" << new_name << ") called");
+
+ if (path.empty()) {
+ handler(Status::InvalidArgument("RenameSnapshot: argument 'path' cannot be empty"));
+ return;
+ }
+ if (old_name.empty()) {
+ handler(Status::InvalidArgument("RenameSnapshot: argument 'old_name' cannot be empty"));
+ return;
+ }
+ if (new_name.empty()) {
+ handler(Status::InvalidArgument("RenameSnapshot: argument 'new_name' cannot be empty"));
+ return;
+ }
+
+ RenameSnapshotRequestProto req;
+ req.set_snapshotroot(path);
+ req.set_snapshotoldname(old_name);
+ req.set_snapshotnewname(new_name);
+
+ auto resp = std::make_shared<RenameSnapshotResponseProto>();
+
+ namenode_.RenameSnapshot(&req, resp,
+ [handler](const Status &stat) {
+ handler(stat);
+ });
+}
+
void NameNodeOperations::AllowSnapshot(const std::string & path, std::function<void(const Status &)> handler) {
using ::hadoop::hdfs::AllowSnapshotRequestProto;
using ::hadoop::hdfs::AllowSnapshotResponseProto;
@@ -621,6 +698,17 @@ void NameNodeOperations::HdfsFileStatusProtoToStatInfo(
stat_info.children_num = fs.childrennum();
}
+void NameNodeOperations::ContentSummaryProtoToContentSummary(
+ hdfs::ContentSummary & content_summary,
+ const ::hadoop::hdfs::ContentSummaryProto & csp) {
+ content_summary.length = csp.length();
+ content_summary.filecount = csp.filecount();
+ content_summary.directorycount = csp.directorycount();
+ content_summary.quota = csp.quota();
+ content_summary.spaceconsumed = csp.spaceconsumed();
+ content_summary.spacequota = csp.spacequota();
+}
+
void NameNodeOperations::GetFsStatsResponseProtoToFsInfo(
hdfs::FsInfo & fs_info,
const std::shared_ptr<::hadoop::hdfs::GetFsStatsResponseProto> & fs) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
index 59b3512..e289d5e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
@@ -21,6 +21,7 @@
#include "rpc/rpc_engine.h"
#include "hdfspp/statinfo.h"
#include "hdfspp/fsinfo.h"
+#include "hdfspp/content_summary.h"
#include "common/namenode_info.h"
#include "ClientNamenodeProtocol.pb.h"
#include "ClientNamenodeProtocol.hrpc.inl"
@@ -67,7 +68,10 @@ public:
std::function<void(const Status &)> handler);
void GetFileInfo(const std::string & path,
- std::function<void(const Status &, const StatInfo &)> handler);
+ std::function<void(const Status &, const StatInfo &)> handler);
+
+ void GetContentSummary(const std::string & path,
+ std::function<void(const Status &, const ContentSummary &)> handler);
void GetFsStats(std::function<void(const Status &, const FsInfo &)> handler);
@@ -97,6 +101,9 @@ public:
void DeleteSnapshot(const std::string & path, const std::string & name,
std::function<void(const Status &)> handler);
+ void RenameSnapshot(const std::string & path, const std::string & old_name, const std::string & new_name,
+ std::function<void(const Status &)> handler);
+
void AllowSnapshot(const std::string & path,
std::function<void(const Status &)> handler);
@@ -107,6 +114,7 @@ public:
private:
static void HdfsFileStatusProtoToStatInfo(hdfs::StatInfo & si, const ::hadoop::hdfs::HdfsFileStatusProto & fs);
+ static void ContentSummaryProtoToContentSummary(hdfs::ContentSummary & content_summary, const ::hadoop::hdfs::ContentSummaryProto & csp);
static void DirectoryListingProtoToStatInfo(std::shared_ptr<std::vector<StatInfo>> stat_infos, const ::hadoop::hdfs::DirectoryListingProto & dl);
static void GetFsStatsResponseProtoToFsInfo(hdfs::FsInfo & fs_info, const std::shared_ptr<::hadoop::hdfs::GetFsStatsResponseProto> & fs);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc
index 7d94944..19d95b4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc
@@ -121,6 +121,8 @@ TEST_F(HdfsExtTest, TestSnapshotOperations) {
EXPECT_EQ((int) std::errc::invalid_argument, errno);
EXPECT_EQ(-1, hdfsDeleteSnapshot(fs, nullptr, "Bad"));
EXPECT_EQ((int) std::errc::invalid_argument, errno);
+ EXPECT_EQ(-1, hdfsRenameSnapshot(fs, nullptr, "Bad", "Bad"));
+ EXPECT_EQ((int) std::errc::invalid_argument, errno);
EXPECT_EQ(-1, hdfsDisallowSnapshot(fs, nullptr));
EXPECT_EQ((int) std::errc::invalid_argument, errno);
@@ -136,6 +138,8 @@ TEST_F(HdfsExtTest, TestSnapshotOperations) {
EXPECT_EQ((int) std::errc::no_such_file_or_directory, errno);
EXPECT_EQ(-1, hdfsDeleteSnapshot(fs, path.c_str(), "Bad"));
EXPECT_EQ((int) std::errc::no_such_file_or_directory, errno);
+ EXPECT_EQ(-1, hdfsRenameSnapshot(fs, path.c_str(), "Bad", "Bad"));
+ EXPECT_EQ((int) std::errc::no_such_file_or_directory, errno);
EXPECT_EQ(-1, hdfsDisallowSnapshot(fs, path.c_str()));
EXPECT_EQ((int) std::errc::no_such_file_or_directory, errno);
@@ -147,6 +151,8 @@ TEST_F(HdfsExtTest, TestSnapshotOperations) {
EXPECT_EQ((int) std::errc::not_a_directory, errno);
EXPECT_EQ(-1, hdfsDeleteSnapshot(fs, path.c_str(), "Bad"));
EXPECT_EQ((int) std::errc::not_a_directory, errno);
+ EXPECT_EQ(-1, hdfsRenameSnapshot(fs, path.c_str(), "Bad", "Bad"));
+ EXPECT_EQ((int) std::errc::not_a_directory, errno);
EXPECT_EQ(-1, hdfsDisallowSnapshot(fs, path.c_str()));
EXPECT_EQ((int) std::errc::not_a_directory, errno);
@@ -167,8 +173,11 @@ TEST_F(HdfsExtTest, TestSnapshotOperations) {
EXPECT_STREQ("Good", file_infos[0].mName);
hdfsFreeFileInfo(file_infos, 1);
+ //Verify snapshot renamed
+ EXPECT_EQ(0, hdfsRenameSnapshot(fs, dirName.c_str(), "Good", "Best"));
+
//Verify snapshot deleted
- EXPECT_EQ(0, hdfsDeleteSnapshot(fs, dirName.c_str(), "Good"));
+ EXPECT_EQ(0, hdfsDeleteSnapshot(fs, dirName.c_str(), "Best"));
EXPECT_EQ(nullptr, file_infos = hdfsListDirectory(fs, snapDir.c_str(), &size));
EXPECT_EQ(0, size);
hdfsFreeFileInfo(file_infos, 0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c
index ddba67f..1edfedc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c
@@ -504,6 +504,10 @@ int hdfsDeleteSnapshot(hdfsFS fs, const char* path, const char* name) {
return libhdfspp_hdfsDeleteSnapshot(fs->libhdfsppRep, path, name);
}
+int hdfsRenameSnapshot(hdfsFS fs, const char* path, const char* old_name, const char* new_name) {
+ return libhdfspp_hdfsRenameSnapshot(fs->libhdfsppRep, path, old_name, new_name);
+}
+
int hdfsAllowSnapshot(hdfsFS fs, const char* path) {
return libhdfspp_hdfsAllowSnapshot(fs->libhdfsppRep, path);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h
index 644ff13..5a67fce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h
@@ -97,5 +97,6 @@
#undef hdfsFind
#undef hdfsCreateSnapshot
#undef hdfsDeleteSnapshot
+#undef hdfsRenameSnapshot
#undef hdfsAllowSnapshot
#undef hdfsDisallowSnapshot
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h
index c186d63..c9bed90 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h
@@ -97,5 +97,6 @@
#define hdfsFind libhdfspp_hdfsFind
#define hdfsCreateSnapshot libhdfspp_hdfsCreateSnapshot
#define hdfsDeleteSnapshot libhdfspp_hdfsDeleteSnapshot
+#define hdfsRenameSnapshot libhdfspp_hdfsRenameSnapshot
#define hdfsAllowSnapshot libhdfspp_hdfsAllowSnapshot
#define hdfsDisallowSnapshot libhdfspp_hdfsDisallowSnapshot
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt
index f0817eb..4a46c7a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt
@@ -23,20 +23,71 @@ set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX})
include_directories( ${LIBHDFSPP_DIR}/include )
link_directories( ${LIBHDFSPP_DIR}/lib )
-add_library(tools_common_obj OBJECT tools_common.cpp)
+add_library(tools_common_obj OBJECT tools_common.cc)
add_library(tools_common $<TARGET_OBJECTS:tools_common_obj>)
-add_executable(hdfs_cat hdfs_cat.cpp)
+add_executable(hdfs_cat hdfs_cat.cc)
target_link_libraries(hdfs_cat tools_common hdfspp)
-add_executable(hdfs_chgrp hdfs_chgrp.cpp)
+add_executable(hdfs_chgrp hdfs_chgrp.cc)
target_link_libraries(hdfs_chgrp tools_common hdfspp)
-add_executable(hdfs_chown hdfs_chown.cpp)
+add_executable(hdfs_chown hdfs_chown.cc)
target_link_libraries(hdfs_chown tools_common hdfspp)
-add_executable(hdfs_chmod hdfs_chmod.cpp)
+add_executable(hdfs_chmod hdfs_chmod.cc)
target_link_libraries(hdfs_chmod tools_common hdfspp)
-add_executable(hdfs_find hdfs_find.cpp)
-target_link_libraries(hdfs_find tools_common hdfspp)
\ No newline at end of file
+add_executable(hdfs_find hdfs_find.cc)
+target_link_libraries(hdfs_find tools_common hdfspp)
+
+add_executable(hdfs_mkdir hdfs_mkdir.cc)
+target_link_libraries(hdfs_mkdir tools_common hdfspp)
+
+add_executable(hdfs_rm hdfs_rm.cc)
+target_link_libraries(hdfs_rm tools_common hdfspp)
+
+add_executable(hdfs_ls hdfs_ls.cc)
+target_link_libraries(hdfs_ls tools_common hdfspp)
+
+add_executable(hdfs_stat hdfs_stat.cc)
+target_link_libraries(hdfs_stat tools_common hdfspp)
+
+add_executable(hdfs_count hdfs_count.cc)
+target_link_libraries(hdfs_count tools_common hdfspp)
+
+add_executable(hdfs_df hdfs_df.cc)
+target_link_libraries(hdfs_df tools_common hdfspp)
+
+add_executable(hdfs_du hdfs_du.cc)
+target_link_libraries(hdfs_du tools_common hdfspp)
+
+add_executable(hdfs_get hdfs_get.cc)
+target_link_libraries(hdfs_get tools_common hdfspp)
+
+add_executable(hdfs_copyToLocal hdfs_copyToLocal.cc)
+target_link_libraries(hdfs_copyToLocal tools_common hdfspp)
+
+add_executable(hdfs_moveToLocal hdfs_moveToLocal.cc)
+target_link_libraries(hdfs_moveToLocal tools_common hdfspp)
+
+add_executable(hdfs_setrep hdfs_setrep.cc)
+target_link_libraries(hdfs_setrep tools_common hdfspp)
+
+add_executable(hdfs_allowSnapshot hdfs_allowSnapshot.cc)
+target_link_libraries(hdfs_allowSnapshot tools_common hdfspp)
+
+add_executable(hdfs_disallowSnapshot hdfs_disallowSnapshot.cc)
+target_link_libraries(hdfs_disallowSnapshot tools_common hdfspp)
+
+add_executable(hdfs_createSnapshot hdfs_createSnapshot.cc)
+target_link_libraries(hdfs_createSnapshot tools_common hdfspp)
+
+add_executable(hdfs_renameSnapshot hdfs_renameSnapshot.cc)
+target_link_libraries(hdfs_renameSnapshot tools_common hdfspp)
+
+add_executable(hdfs_deleteSnapshot hdfs_deleteSnapshot.cc)
+target_link_libraries(hdfs_deleteSnapshot tools_common hdfspp)
+
+add_executable(hdfs_tail hdfs_tail.cc)
+target_link_libraries(hdfs_tail tools_common hdfspp)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_allowSnapshot.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_allowSnapshot.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_allowSnapshot.cc
new file mode 100644
index 0000000..00709e4
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_allowSnapshot.cc
@@ -0,0 +1,90 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_allowSnapshot [OPTION] PATH"
+ << std::endl
+ << std::endl << "Allowing snapshots of a directory at PATH to be created."
+ << std::endl << "If the operation completes successfully, the directory becomes snapshottable."
+ << std::endl
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_allowSnapshot hdfs://localhost.localdomain:8020/dir"
+ << std::endl << "hdfs_allowSnapshot /dir1/dir2"
+ << std::endl;
+}
+
+int main(int argc, char *argv[]) {
+ //We should have at least 2 arguments
+ if (argc < 2) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ int input;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "h")) != -1) {
+ switch (input)
+ {
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ case '?':
+ if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+ std::string uri_path = argv[optind];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ hdfs::Status status = fs->AllowSnapshot(uri->get_path());
+ if (!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_cat.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_cat.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_cat.cc
new file mode 100644
index 0000000..ec347cb
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_cat.cc
@@ -0,0 +1,87 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_cat [OPTION] FILE"
+ << std::endl
+ << std::endl << "Concatenate FILE to standard output."
+ << std::endl
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_cat hdfs://localhost.localdomain:8020/dir/file"
+ << std::endl << "hdfs_cat /dir/file"
+ << std::endl;
+}
+
+#define BUF_SIZE 4096
+
+int main(int argc, char *argv[]) {
+ if (argc != 2) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ int input;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "h")) != -1) {
+ switch (input)
+ {
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ case '?':
+ if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ std::string uri_path = argv[optind];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ readFile(fs, uri->get_path(), 0, stdout, false);
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_cat.cpp
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_cat.cpp b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_cat.cpp
deleted file mode 100644
index 166a7bf..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_cat.cpp
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include "tools_common.h"
-
-void usage(){
- std::cout << "Usage: hdfs_cat [OPTION] FILE"
- << std::endl
- << std::endl << "Concatenate FILE to standard output."
- << std::endl
- << std::endl << " -h display this help and exit"
- << std::endl
- << std::endl << "Examples:"
- << std::endl << "hdfs_cat hdfs://localhost.localdomain:9433/dir/file"
- << std::endl << "hdfs_cat /dir/file"
- << std::endl;
-}
-
-#define BUF_SIZE 4096
-
-int main(int argc, char *argv[]) {
- if (argc != 2) {
- usage();
- exit(EXIT_FAILURE);
- }
-
- int input;
-
- //Using GetOpt to read in the values
- opterr = 0;
- while ((input = getopt(argc, argv, "h")) != -1) {
- switch (input)
- {
- case 'h':
- usage();
- exit(EXIT_SUCCESS);
- break;
- case '?':
- if (isprint(optopt))
- std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
- else
- std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
- usage();
- exit(EXIT_FAILURE);
- default:
- exit(EXIT_FAILURE);
- }
- }
-
- std::string uri_path = argv[optind];
-
- //Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
-
- //TODO: HDFS-9539 Currently options can be returned empty
- hdfs::Options options = *hdfs::getOptions();
-
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), options);
- if (!fs) {
- std::cerr << "Could not connect the file system. " << std::endl;
- exit(EXIT_FAILURE);
- }
-
- hdfs::FileHandle *file_raw = nullptr;
- hdfs::Status status = fs->Open(uri->get_path(), &file_raw);
- if (!status.ok()) {
- std::cerr << "Could not open file " << uri->get_path() << ". " << status.ToString() << std::endl;
- exit(EXIT_FAILURE);
- }
- //wrapping file_raw into a unique pointer to guarantee deletion
- std::unique_ptr<hdfs::FileHandle> file(file_raw);
-
- char input_buffer[BUF_SIZE];
- ssize_t total_bytes_read = 0;
- size_t last_bytes_read = 0;
-
- do{
- //Reading file chunks
- status = file->Read(input_buffer, sizeof(input_buffer), &last_bytes_read);
- if(status.ok()) {
- //Writing file chunks to stdout
- fwrite(input_buffer, last_bytes_read, 1, stdout);
- total_bytes_read += last_bytes_read;
- } else {
- if(status.is_invalid_offset()){
- //Reached the end of the file
- break;
- } else {
- std::cerr << "Error reading the file: " << status.ToString() << std::endl;
- exit(EXIT_FAILURE);
- }
- }
- } while (last_bytes_read > 0);
-
- // Clean up static data and prevent valgrind memory leaks
- google::protobuf::ShutdownProtobufLibrary();
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cc
new file mode 100644
index 0000000..3b3cd50
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cc
@@ -0,0 +1,189 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include <future>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_chgrp [OPTION] GROUP FILE"
+ << std::endl
+ << std::endl << "Change the group association of each FILE to GROUP."
+ << std::endl << "The user must be the owner of files. Additional information is in the Permissions Guide:"
+ << std::endl << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html"
+ << std::endl
+ << std::endl << " -R operate on files and directories recursively"
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_chgrp -R new_group hdfs://localhost.localdomain:8020/dir/file"
+ << std::endl << "hdfs_chgrp new_group /dir/file"
+ << std::endl;
+}
+
+struct SetOwnerState {
+ const std::string username;
+ const std::string groupname;
+ const std::function<void(const hdfs::Status &)> handler;
+ //The request counter is incremented once every time SetOwner async call is made
+ uint64_t request_counter;
+ //This boolean will be set when find returns the last result
+ bool find_is_done;
+ //Final status to be returned
+ hdfs::Status status;
+ //Shared variables will need protection with a lock
+ std::mutex lock;
+ SetOwnerState(const std::string & username_, const std::string & groupname_,
+ const std::function<void(const hdfs::Status &)> & handler_,
+ uint64_t request_counter_, bool find_is_done_)
+ : username(username_),
+ groupname(groupname_),
+ handler(handler_),
+ request_counter(request_counter_),
+ find_is_done(find_is_done_),
+ status(),
+ lock() {
+ }
+};
+
+int main(int argc, char *argv[]) {
+ //We should have 3 or 4 parameters
+ if (argc != 3 && argc != 4) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ bool recursive = false;
+ int input;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "Rh")) != -1) {
+ switch (input)
+ {
+ case 'R':
+ recursive = 1;
+ break;
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ case '?':
+ if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+ std::string group = argv[optind];
+ //Owner stays the same, just group association changes.
+ std::string owner = "";
+ std::string uri_path = argv[optind + 1];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), true);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ /* wrap async FileSystem::SetOwner with promise to make it a blocking call */
+ std::shared_ptr<std::promise<hdfs::Status>> promise = std::make_shared<std::promise<hdfs::Status>>();
+ std::future<hdfs::Status> future(promise->get_future());
+ auto handler = [promise](const hdfs::Status &s) {
+ promise->set_value(s);
+ };
+
+ if(!recursive){
+ fs->SetOwner(uri->get_path(), owner, group, handler);
+ }
+ else {
+ //Allocating shared state, which includes:
+ //username and groupname to be set, handler to be called, request counter, and a boolean to keep track if find is done
+ std::shared_ptr<SetOwnerState> state = std::make_shared<SetOwnerState>(owner, group, handler, 0, false);
+
+ // Keep requesting more from Find until we process the entire listing. Call handler when Find is done and reques counter is 0.
+ // Find guarantees that the handler will only be called once at a time so we do not need locking in handlerFind.
+ auto handlerFind = [fs, state](const hdfs::Status &status_find, const std::vector<hdfs::StatInfo> & stat_infos, bool has_more_results) -> bool {
+
+ //For each result returned by Find we call async SetOwner with the handler below.
+ //SetOwner DOES NOT guarantee that the handler will only be called once at a time, so we DO need locking in handlerSetOwner.
+ auto handlerSetOwner = [state](const hdfs::Status &status_set_owner) {
+ std::lock_guard<std::mutex> guard(state->lock);
+
+ //Decrement the counter once since we are done with this async call
+ if (!status_set_owner.ok() && state->status.ok()){
+ //We make sure we set state->status only on the first error.
+ state->status = status_set_owner;
+ }
+ state->request_counter--;
+ if(state->request_counter == 0 && state->find_is_done){
+ state->handler(state->status); //exit
+ }
+ };
+ if(!stat_infos.empty() && state->status.ok()) {
+ for (hdfs::StatInfo const& s : stat_infos) {
+ //Launch an asynchronous call to SetOwner for every returned result
+ state->request_counter++;
+ fs->SetOwner(s.full_path, state->username, state->groupname, handlerSetOwner);
+ }
+ }
+
+ //Lock this section because handlerSetOwner might be accessing the same
+ //shared variables simultaneously
+ std::lock_guard<std::mutex> guard(state->lock);
+ if (!status_find.ok() && state->status.ok()){
+ //We make sure we set state->status only on the first error.
+ state->status = status_find;
+ }
+ if(!has_more_results){
+ state->find_is_done = true;
+ if(state->request_counter == 0){
+ state->handler(state->status); //exit
+ }
+ return false;
+ }
+ return true;
+ };
+
+ //Asynchronous call to Find
+ fs->Find(uri->get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
+ }
+
+ /* block until promise is set */
+ hdfs::Status status = future.get();
+ if (!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cpp
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cpp b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cpp
deleted file mode 100644
index 2bb6843..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cpp
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include <future>
-#include "tools_common.h"
-
-void usage(){
- std::cout << "Usage: hdfs_chgrp [OPTION] GROUP FILE"
- << std::endl
- << std::endl << "Change the group association of each FILE to GROUP."
- << std::endl << "The user must be the owner of files. Additional information is in the Permissions Guide:"
- << std::endl << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html"
- << std::endl
- << std::endl << " -R operate on files and directories recursively"
- << std::endl << " -h display this help and exit"
- << std::endl
- << std::endl << "Examples:"
- << std::endl << "hdfs_chgrp -R new_group hdfs://localhost.localdomain:9433/dir/file"
- << std::endl << "hdfs_chgrp new_group /dir/file"
- << std::endl;
-}
-
-struct SetOwnerState {
- const std::string username;
- const std::string groupname;
- const std::function<void(const hdfs::Status &)> handler;
- //The request counter is incremented once every time SetOwner async call is made
- uint64_t request_counter;
- //This boolean will be set when find returns the last result
- bool find_is_done;
- //Final status to be returned
- hdfs::Status status;
- //Shared variables will need protection with a lock
- std::mutex lock;
- SetOwnerState(const std::string & username_, const std::string & groupname_,
- const std::function<void(const hdfs::Status &)> & handler_,
- uint64_t request_counter_, bool find_is_done_)
- : username(username_),
- groupname(groupname_),
- handler(handler_),
- request_counter(request_counter_),
- find_is_done(find_is_done_),
- status(),
- lock() {
- }
-};
-
-int main(int argc, char *argv[]) {
- //We should have 3 or 4 parameters
- if (argc != 3 && argc != 4) {
- usage();
- exit(EXIT_FAILURE);
- }
-
- bool recursive = false;
- int input;
-
- //Using GetOpt to read in the values
- opterr = 0;
- while ((input = getopt(argc, argv, "Rh")) != -1) {
- switch (input)
- {
- case 'R':
- recursive = 1;
- break;
- case 'h':
- usage();
- exit(EXIT_SUCCESS);
- break;
- case '?':
- if (isprint(optopt))
- std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
- else
- std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
- usage();
- exit(EXIT_FAILURE);
- default:
- exit(EXIT_FAILURE);
- }
- }
- std::string group = argv[optind];
- //Owner stays the same, just group association changes.
- std::string owner = "";
- std::string uri_path = argv[optind + 1];
-
- //Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
-
- //TODO: HDFS-9539 Currently options can be returned empty
- hdfs::Options options = *hdfs::getOptions();
-
- //TODO: HDFS-9539 - until then we increase the time-out to allow all recursive async calls to finish
- options.rpc_timeout = std::numeric_limits<int>::max();
-
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), options);
- if (!fs) {
- std::cerr << "Could not connect the file system. " << std::endl;
- exit(EXIT_FAILURE);
- }
-
- /* wrap async FileSystem::SetOwner with promise to make it a blocking call */
- std::shared_ptr<std::promise<hdfs::Status>> promise = std::make_shared<std::promise<hdfs::Status>>();
- std::future<hdfs::Status> future(promise->get_future());
- auto handler = [promise](const hdfs::Status &s) {
- promise->set_value(s);
- };
-
- if(!recursive){
- fs->SetOwner(uri->get_path(), owner, group, handler);
- }
- else {
- //Allocating shared state, which includes:
- //username and groupname to be set, handler to be called, request counter, and a boolean to keep track if find is done
- std::shared_ptr<SetOwnerState> state = std::make_shared<SetOwnerState>(owner, group, handler, 0, false);
-
- // Keep requesting more from Find until we process the entire listing. Call handler when Find is done and reques counter is 0.
- // Find guarantees that the handler will only be called once at a time so we do not need locking in handlerFind.
- auto handlerFind = [fs, state](const hdfs::Status &status_find, const std::vector<hdfs::StatInfo> & stat_infos, bool has_more_results) -> bool {
-
- //For each result returned by Find we call async SetOwner with the handler below.
- //SetOwner DOES NOT guarantee that the handler will only be called once at a time, so we DO need locking in handlerSetOwner.
- auto handlerSetOwner = [state](const hdfs::Status &status_set_owner) {
- std::lock_guard<std::mutex> guard(state->lock);
-
- //Decrement the counter once since we are done with this async call
- if (!status_set_owner.ok() && state->status.ok()){
- //We make sure we set state->status only on the first error.
- state->status = status_set_owner;
- }
- state->request_counter--;
- if(state->request_counter == 0 && state->find_is_done){
- state->handler(state->status); //exit
- }
- };
- if(!stat_infos.empty() && state->status.ok()) {
- for (hdfs::StatInfo const& s : stat_infos) {
- //Launch an asynchronous call to SetOwner for every returned result
- state->request_counter++;
- fs->SetOwner(s.full_path, state->username, state->groupname, handlerSetOwner);
- }
- }
-
- //Lock this section because handlerSetOwner might be accessing the same
- //shared variables simultaneously
- std::lock_guard<std::mutex> guard(state->lock);
- if (!status_find.ok() && state->status.ok()){
- //We make sure we set state->status only on the first error.
- state->status = status_find;
- }
- if(!has_more_results){
- state->find_is_done = true;
- if(state->request_counter == 0){
- state->handler(state->status); //exit
- }
- return false;
- }
- return true;
- };
-
- //Asynchronous call to Find
- fs->Find(uri->get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
- }
-
- /* block until promise is set */
- hdfs::Status status = future.get();
- if (!status.ok()) {
- std::cerr << "Error: " << status.ToString() << std::endl;
- exit(EXIT_FAILURE);
- }
-
- // Clean up static data and prevent valgrind memory leaks
- google::protobuf::ShutdownProtobufLibrary();
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cc
new file mode 100644
index 0000000..69d61a5
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cc
@@ -0,0 +1,187 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include <future>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_chmod [OPTION] <MODE[,MODE]... | OCTALMODE> FILE"
+ << std::endl
+ << std::endl << "Change the permissions of each FILE to MODE."
+ << std::endl << "The user must be the owner of the file, or else a super-user."
+ << std::endl << "Additional information is in the Permissions Guide:"
+ << std::endl << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html"
+ << std::endl
+ << std::endl << " -R operate on files and directories recursively"
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_chmod -R 755 hdfs://localhost.localdomain:8020/dir/file"
+ << std::endl << "hdfs_chmod 777 /dir/file"
+ << std::endl;
+}
+
+struct SetPermissionState {
+ const uint16_t permissions;
+ const std::function<void(const hdfs::Status &)> handler;
+ //The request counter is incremented once every time SetOwner async call is made
+ uint64_t request_counter;
+ //This boolean will be set when find returns the last result
+ bool find_is_done;
+ //Final status to be returned
+ hdfs::Status status;
+ //Shared variables will need protection with a lock
+ std::mutex lock;
+ SetPermissionState(const uint16_t permissions_, const std::function<void(const hdfs::Status &)> & handler_,
+ uint64_t request_counter_, bool find_is_done_)
+ : permissions(permissions_),
+ handler(handler_),
+ request_counter(request_counter_),
+ find_is_done(find_is_done_),
+ status(),
+ lock() {
+ }
+};
+
+int main(int argc, char *argv[]) {
+ //We should have 3 or 4 parameters
+ if (argc != 3 && argc != 4) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ bool recursive = false;
+ int input;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "Rh")) != -1) {
+ switch (input)
+ {
+ case 'R':
+ recursive = 1;
+ break;
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ case '?':
+ if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+ std::string permissions = argv[optind];
+ std::string uri_path = argv[optind + 1];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), true);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ /* wrap async FileSystem::SetPermission with promise to make it a blocking call */
+ std::shared_ptr<std::promise<hdfs::Status>> promise = std::make_shared<std::promise<hdfs::Status>>();
+ std::future<hdfs::Status> future(promise->get_future());
+ auto handler = [promise](const hdfs::Status &s) {
+ promise->set_value(s);
+ };
+
+ //strtol() is reading the value with base 8, NULL because we are reading in just one value.
+ uint16_t perm = strtol(permissions.c_str(), NULL, 8);
+ if(!recursive){
+ fs->SetPermission(uri->get_path(), perm, handler);
+ }
+ else {
+ //Allocating shared state, which includes:
+ //permissions to be set, handler to be called, request counter, and a boolean to keep track if find is done
+ std::shared_ptr<SetPermissionState> state = std::make_shared<SetPermissionState>(perm, handler, 0, false);
+
+ // Keep requesting more from Find until we process the entire listing. Call handler when Find is done and reques counter is 0.
+ // Find guarantees that the handler will only be called once at a time so we do not need locking in handlerFind.
+ auto handlerFind = [fs, state](const hdfs::Status &status_find, const std::vector<hdfs::StatInfo> & stat_infos, bool has_more_results) -> bool {
+
+ //For each result returned by Find we call async SetPermission with the handler below.
+ //SetPermission DOES NOT guarantee that the handler will only be called once at a time, so we DO need locking in handlerSetPermission.
+ auto handlerSetPermission = [state](const hdfs::Status &status_set_permission) {
+ std::lock_guard<std::mutex> guard(state->lock);
+
+ //Decrement the counter once since we are done with this async call
+ if (!status_set_permission.ok() && state->status.ok()){
+ //We make sure we set state->status only on the first error.
+ state->status = status_set_permission;
+ }
+ state->request_counter--;
+ if(state->request_counter == 0 && state->find_is_done){
+ state->handler(state->status); //exit
+ }
+ };
+ if(!stat_infos.empty() && state->status.ok()) {
+ for (hdfs::StatInfo const& s : stat_infos) {
+ //Launch an asynchronous call to SetPermission for every returned result
+ state->request_counter++;
+ fs->SetPermission(s.full_path, state->permissions, handlerSetPermission);
+ }
+ }
+
+ //Lock this section because handlerSetPermission might be accessing the same
+ //shared variables simultaneously
+ std::lock_guard<std::mutex> guard(state->lock);
+ if (!status_find.ok() && state->status.ok()){
+ //We make sure we set state->status only on the first error.
+ state->status = status_find;
+ }
+ if(!has_more_results){
+ state->find_is_done = true;
+ if(state->request_counter == 0){
+ state->handler(state->status); //exit
+ }
+ return false;
+ }
+ return true;
+ };
+
+ //Asynchronous call to Find
+ fs->Find(uri->get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
+ }
+
+ /* block until promise is set */
+ hdfs::Status status = future.get();
+ if (!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cpp
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cpp b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cpp
deleted file mode 100644
index 0a001d6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cpp
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include <future>
-#include "tools_common.h"
-
-void usage(){
- std::cout << "Usage: hdfs_chmod [OPTION] <MODE[,MODE]... | OCTALMODE> FILE"
- << std::endl
- << std::endl << "Change the permissions of each FILE to MODE."
- << std::endl << "The user must be the owner of the file, or else a super-user."
- << std::endl << "Additional information is in the Permissions Guide:"
- << std::endl << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html"
- << std::endl
- << std::endl << " -R operate on files and directories recursively"
- << std::endl << " -h display this help and exit"
- << std::endl
- << std::endl << "Examples:"
- << std::endl << "hdfs_chmod -R 755 hdfs://localhost.localdomain:9433/dir/file"
- << std::endl << "hdfs_chmod 777 /dir/file"
- << std::endl;
-}
-
-struct SetPermissionState {
- const uint16_t permissions;
- const std::function<void(const hdfs::Status &)> handler;
- //The request counter is incremented once every time SetOwner async call is made
- uint64_t request_counter;
- //This boolean will be set when find returns the last result
- bool find_is_done;
- //Final status to be returned
- hdfs::Status status;
- //Shared variables will need protection with a lock
- std::mutex lock;
- SetPermissionState(const uint16_t permissions_, const std::function<void(const hdfs::Status &)> & handler_,
- uint64_t request_counter_, bool find_is_done_)
- : permissions(permissions_),
- handler(handler_),
- request_counter(request_counter_),
- find_is_done(find_is_done_),
- status(),
- lock() {
- }
-};
-
-int main(int argc, char *argv[]) {
- //We should have 3 or 4 parameters
- if (argc != 3 && argc != 4) {
- usage();
- exit(EXIT_FAILURE);
- }
-
- bool recursive = false;
- int input;
-
- //Using GetOpt to read in the values
- opterr = 0;
- while ((input = getopt(argc, argv, "Rh")) != -1) {
- switch (input)
- {
- case 'R':
- recursive = 1;
- break;
- case 'h':
- usage();
- exit(EXIT_SUCCESS);
- break;
- case '?':
- if (isprint(optopt))
- std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
- else
- std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
- usage();
- exit(EXIT_FAILURE);
- default:
- exit(EXIT_FAILURE);
- }
- }
- std::string permissions = argv[optind];
- std::string uri_path = argv[optind + 1];
-
- //Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
-
- //TODO: HDFS-9539 Currently options can be returned empty
- hdfs::Options options = *hdfs::getOptions();
-
- //TODO: HDFS-9539 - until then we increase the time-out to allow all recursive async calls to finish
- options.rpc_timeout = std::numeric_limits<int>::max();
-
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), options);
- if (!fs) {
- std::cerr << "Could not connect the file system. " << std::endl;
- exit(EXIT_FAILURE);
- }
-
- /* wrap async FileSystem::SetPermission with promise to make it a blocking call */
- std::shared_ptr<std::promise<hdfs::Status>> promise = std::make_shared<std::promise<hdfs::Status>>();
- std::future<hdfs::Status> future(promise->get_future());
- auto handler = [promise](const hdfs::Status &s) {
- promise->set_value(s);
- };
-
- //strtol() is reading the value with base 8, NULL because we are reading in just one value.
- uint16_t perm = strtol(permissions.c_str(), NULL, 8);
- if(!recursive){
- fs->SetPermission(uri->get_path(), perm, handler);
- }
- else {
- //Allocating shared state, which includes:
- //username and groupname to be set, handler to be called, request counter, and a boolean to keep track if find is done
- std::shared_ptr<SetPermissionState> state = std::make_shared<SetPermissionState>(perm, handler, 0, false);
-
- // Keep requesting more from Find until we process the entire listing. Call handler when Find is done and reques counter is 0.
- // Find guarantees that the handler will only be called once at a time so we do not need locking in handlerFind.
- auto handlerFind = [fs, state](const hdfs::Status &status_find, const std::vector<hdfs::StatInfo> & stat_infos, bool has_more_results) -> bool {
-
- //For each result returned by Find we call async SetOwner with the handler below.
- //SetOwner DOES NOT guarantee that the handler will only be called once at a time, so we DO need locking in handlerSetOwner.
- auto handlerSetOwner = [state](const hdfs::Status &status_set_owner) {
- std::lock_guard<std::mutex> guard(state->lock);
-
- //Decrement the counter once since we are done with this async call
- if (!status_set_owner.ok() && state->status.ok()){
- //We make sure we set state->status only on the first error.
- state->status = status_set_owner;
- }
- state->request_counter--;
- if(state->request_counter == 0 && state->find_is_done){
- state->handler(state->status); //exit
- }
- };
- if(!stat_infos.empty() && state->status.ok()) {
- for (hdfs::StatInfo const& s : stat_infos) {
- //Launch an asynchronous call to SetOwner for every returned result
- state->request_counter++;
- fs->SetPermission(s.full_path, state->permissions, handlerSetOwner);
- }
- }
-
- //Lock this section because handlerSetOwner might be accessing the same
- //shared variables simultaneously
- std::lock_guard<std::mutex> guard(state->lock);
- if (!status_find.ok() && state->status.ok()){
- //We make sure we set state->status only on the first error.
- state->status = status_find;
- }
- if(!has_more_results){
- state->find_is_done = true;
- if(state->request_counter == 0){
- state->handler(state->status); //exit
- }
- return false;
- }
- return true;
- };
-
- //Asynchronous call to Find
- fs->Find(uri->get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
- }
-
- /* block until promise is set */
- hdfs::Status status = future.get();
- if (!status.ok()) {
- std::cerr << "Error: " << status.ToString() << std::endl;
- exit(EXIT_FAILURE);
- }
-
- // Clean up static data and prevent valgrind memory leaks
- google::protobuf::ShutdownProtobufLibrary();
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cc
new file mode 100644
index 0000000..59ff9c3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cc
@@ -0,0 +1,199 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include <future>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_chown [OPTION] [OWNER][:[GROUP]] FILE"
+ << std::endl
+ << std::endl << "Change the owner and/or group of each FILE to OWNER and/or GROUP."
+ << std::endl << "The user must be a super-user. Additional information is in the Permissions Guide:"
+ << std::endl << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html"
+ << std::endl
+ << std::endl << " -R operate on files and directories recursively"
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Owner is unchanged if missing. Group is unchanged if missing."
+ << std::endl << "OWNER and GROUP may be numeric as well as symbolic."
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_chown -R new_owner:new_group hdfs://localhost.localdomain:8020/dir/file"
+ << std::endl << "hdfs_chown new_owner /dir/file"
+ << std::endl;
+}
+
+struct SetOwnerState {
+ const std::string username;
+ const std::string groupname;
+ const std::function<void(const hdfs::Status &)> handler;
+ //The request counter is incremented once every time SetOwner async call is made
+ uint64_t request_counter;
+ //This boolean will be set when find returns the last result
+ bool find_is_done;
+ //Final status to be returned
+ hdfs::Status status;
+ //Shared variables will need protection with a lock
+ std::mutex lock;
+ SetOwnerState(const std::string & username_, const std::string & groupname_,
+ const std::function<void(const hdfs::Status &)> & handler_,
+ uint64_t request_counter_, bool find_is_done_)
+ : username(username_),
+ groupname(groupname_),
+ handler(handler_),
+ request_counter(request_counter_),
+ find_is_done(find_is_done_),
+ status(),
+ lock() {
+ }
+};
+
+int main(int argc, char *argv[]) {
+ //We should have 3 or 4 parameters
+ if (argc != 3 && argc != 4) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ bool recursive = false;
+ int input;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "Rh")) != -1) {
+ switch (input)
+ {
+ case 'R':
+ recursive = 1;
+ break;
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ case '?':
+ if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+ std::string owner_and_group = argv[optind];
+ std::string uri_path = argv[optind + 1];
+
+ std::string owner, group;
+ size_t owner_end = owner_and_group.find(":");
+ if(owner_end == std::string::npos) {
+ owner = owner_and_group;
+ } else {
+ owner = owner_and_group.substr(0, owner_end);
+ group = owner_and_group.substr(owner_end + 1);
+ }
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), true);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ /* wrap async FileSystem::SetOwner with promise to make it a blocking call */
+ std::shared_ptr<std::promise<hdfs::Status>> promise = std::make_shared<std::promise<hdfs::Status>>();
+ std::future<hdfs::Status> future(promise->get_future());
+ auto handler = [promise](const hdfs::Status &s) {
+ promise->set_value(s);
+ };
+
+ if(!recursive){
+ fs->SetOwner(uri->get_path(), owner, group, handler);
+ }
+ else {
+ //Allocating shared state, which includes:
+ //username and groupname to be set, handler to be called, request counter, and a boolean to keep track if find is done
+ std::shared_ptr<SetOwnerState> state = std::make_shared<SetOwnerState>(owner, group, handler, 0, false);
+
+ // Keep requesting more from Find until we process the entire listing. Call handler when Find is done and reques counter is 0.
+ // Find guarantees that the handler will only be called once at a time so we do not need locking in handlerFind.
+ auto handlerFind = [fs, state](const hdfs::Status &status_find, const std::vector<hdfs::StatInfo> & stat_infos, bool has_more_results) -> bool {
+
+ //For each result returned by Find we call async SetOwner with the handler below.
+ //SetOwner DOES NOT guarantee that the handler will only be called once at a time, so we DO need locking in handlerSetOwner.
+ auto handlerSetOwner = [state](const hdfs::Status &status_set_owner) {
+ std::lock_guard<std::mutex> guard(state->lock);
+
+ //Decrement the counter once since we are done with this async call
+ if (!status_set_owner.ok() && state->status.ok()){
+ //We make sure we set state->status only on the first error.
+ state->status = status_set_owner;
+ }
+ state->request_counter--;
+ if(state->request_counter == 0 && state->find_is_done){
+ state->handler(state->status); //exit
+ }
+ };
+ if(!stat_infos.empty() && state->status.ok()) {
+ for (hdfs::StatInfo const& s : stat_infos) {
+ //Launch an asynchronous call to SetOwner for every returned result
+ state->request_counter++;
+ fs->SetOwner(s.full_path, state->username, state->groupname, handlerSetOwner);
+ }
+ }
+
+ //Lock this section because handlerSetOwner might be accessing the same
+ //shared variables simultaneously
+ std::lock_guard<std::mutex> guard(state->lock);
+ if (!status_find.ok() && state->status.ok()){
+ //We make sure we set state->status only on the first error.
+ state->status = status_find;
+ }
+ if(!has_more_results){
+ state->find_is_done = true;
+ if(state->request_counter == 0){
+ state->handler(state->status); //exit
+ }
+ return false;
+ }
+ return true;
+ };
+
+ //Asynchronous call to Find
+ fs->Find(uri->get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
+ }
+
+ /* block until promise is set */
+ hdfs::Status status = future.get();
+ if (!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/52] [abbrv] hadoop git commit: HDFS-9271: libhdfs++: Implement
basic NN operations. Contributed by Anatoli Shein.
Posted by jh...@apache.org.
HDFS-9271: libhdfs++: Implement basic NN operations. Contributed by Anatoli Shein.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2a8edd4e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2a8edd4e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2a8edd4e
Branch: refs/heads/HDFS-8707
Commit: 2a8edd4e523a2e2e90ee570d0c61a600cf7bf610
Parents: b1ed72e
Author: James <jh...@apache.org>
Authored: Wed Aug 10 13:27:31 2016 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../libhdfs-tests/test_libhdfs_threaded.c | 9 +-
.../native/libhdfspp/include/hdfspp/hdfs_ext.h | 14 +
.../native/libhdfspp/include/hdfspp/hdfspp.h | 57 ++-
.../native/libhdfspp/include/hdfspp/options.h | 7 +
.../native/libhdfspp/include/hdfspp/status.h | 4 +
.../native/libhdfspp/lib/bindings/c/hdfs.cc | 464 +++++++++++++++++--
.../libhdfspp/lib/common/hdfs_configuration.cc | 1 +
.../libhdfspp/lib/common/hdfs_configuration.h | 1 +
.../main/native/libhdfspp/lib/common/options.cc | 4 +-
.../main/native/libhdfspp/lib/common/status.cc | 4 +
.../main/native/libhdfspp/lib/common/util.cc | 9 +
.../src/main/native/libhdfspp/lib/common/util.h | 2 +
.../main/native/libhdfspp/lib/fs/filehandle.cc | 7 +-
.../main/native/libhdfspp/lib/fs/filehandle.h | 7 +
.../main/native/libhdfspp/lib/fs/filesystem.cc | 153 +++++-
.../main/native/libhdfspp/lib/fs/filesystem.h | 24 +-
.../libhdfspp/lib/fs/namenode_operations.cc | 156 ++++++-
.../libhdfspp/lib/fs/namenode_operations.h | 21 +-
.../native/libhdfspp/tests/hdfs_ext_test.cc | 267 ++++++++++-
.../src/main/native/libhdfspp/tests/hdfs_shim.c | 75 +--
20 files changed, 1156 insertions(+), 130 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a8edd4e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
index 1ab3700..343e05a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
@@ -93,13 +93,12 @@ static int doTestGetDefaultBlockSize(hdfsFS fs, const char *path)
blockSize = hdfsGetDefaultBlockSize(fs);
if (blockSize < 0) {
- ret = errno;
- fprintf(stderr, "hdfsGetDefaultBlockSize failed with error %d\n", ret);
- return ret;
+ fprintf(stderr, "hdfsGetDefaultBlockSize failed with error %d\n", errno);
+ return -1;
} else if (blockSize != TLH_DEFAULT_BLOCK_SIZE) {
fprintf(stderr, "hdfsGetDefaultBlockSize got %"PRId64", but we "
"expected %d\n", blockSize, TLH_DEFAULT_BLOCK_SIZE);
- return EIO;
+ return -1;
}
blockSize = hdfsGetDefaultBlockSizeAtPath(fs, path);
@@ -205,6 +204,8 @@ static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs,
EXPECT_ZERO(hdfsHSync(fs, file));
EXPECT_ZERO(hdfsCloseFile(fs, file));
+ EXPECT_ZERO(doTestGetDefaultBlockSize(fs, paths->file1));
+
/* There should be 1 entry in the directory. */
hdfsFileInfo * dirList = hdfsListDirectory(fs, paths->prefix, &numEntries);
EXPECT_NONNULL(dirList);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a8edd4e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
index 6ec3a4b..ce9f0f5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
@@ -115,6 +115,20 @@ int hdfsBuilderConfGetStr(struct hdfsBuilder *bld, const char *key,
LIBHDFS_EXTERNAL
int hdfsBuilderConfGetInt(struct hdfsBuilder *bld, const char *key, int32_t *val);
+
+/**
+ * Get a configuration long from the settings currently read into the builder.
+ *
+ * @param key The key to find
+ * @param val (out param) The value. This will NOT be changed if the
+ * key isn't found.
+ *
+ * @return 0 on success; -1 otherwise.
+ * Failure to find the key is not an error.
+ */
+LIBHDFS_EXTERNAL
+int hdfsBuilderConfGetLong(struct hdfsBuilder *bld, const char *key, int64_t *val);
+
struct hdfsDNInfo {
const char * ip_address;
const char * hostname;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a8edd4e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
index 20a651a..46fe8e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
@@ -124,6 +124,12 @@ public:
*/
virtual void SetFileEventCallback(file_event_callback callback) = 0;
+ /* how many bytes have been successfully read */
+ virtual uint64_t get_bytes_read() = 0;
+
+ /* resets the number of bytes read to zero */
+ virtual void clear_bytes_read() = 0;
+
virtual ~FileHandle();
};
@@ -172,6 +178,41 @@ class FileSystem {
virtual Status Open(const std::string &path, FileHandle **handle) = 0;
/**
+ * Get the block size for the given file.
+ * @param path The path to the file
+ */
+ virtual void GetPreferredBlockSize(const std::string &path,
+ const std::function<void(const Status &, const uint64_t &)> &handler) = 0;
+ virtual Status GetPreferredBlockSize(const std::string &path, uint64_t & block_size) = 0;
+
+ /**
+ * Set replication for an existing file.
+ * <p>
+ * The NameNode sets replication to the new value and returns.
+ * The actual block replication is not expected to be performed during
+ * this method call. The blocks will be populated or removed in the
+ * background as the result of the routine block maintenance procedures.
+ *
+ * @param src file name
+ * @param replication new replication
+ */
+ virtual void SetReplication(const std::string & path, int16_t replication, std::function<void(const Status &)> handler) = 0;
+ virtual Status SetReplication(const std::string & path, int16_t replication) = 0;
+
+ /**
+ * Sets the modification and access time of the file to the specified time.
+ * @param src The string representation of the path
+ * @param mtime The number of milliseconds since Jan 1, 1970.
+ * Setting mtime to -1 means that modification time should not
+ * be set by this call.
+ * @param atime The number of milliseconds since Jan 1, 1970.
+ * Setting atime to -1 means that access time should not be set
+ * by this call.
+ */
+ virtual void SetTimes(const std::string & path, uint64_t mtime, uint64_t atime, std::function<void(const Status &)> handler) = 0;
+ virtual Status SetTimes(const std::string & path, uint64_t mtime, uint64_t atime) = 0;
+
+ /**
* Returns metadata about the file if the file/directory exists.
**/
virtual void
@@ -209,12 +250,12 @@ class FileSystem {
std::shared_ptr<std::vector<StatInfo>> & stat_infos) = 0;
/**
- * Returns the locations of all known blocks for the indicated file, or an error
+ * Returns the locations of all known blocks for the indicated file (or part of it), or an error
* if the information clould not be found
*/
- virtual void GetBlockLocations(const std::string & path,
+ virtual void GetBlockLocations(const std::string & path, uint64_t offset, uint64_t length,
const std::function<void(const Status &, std::shared_ptr<FileBlockLocation> locations)> ) = 0;
- virtual Status GetBlockLocations(const std::string & path,
+ virtual Status GetBlockLocations(const std::string & path, uint64_t offset, uint64_t length,
std::shared_ptr<FileBlockLocation> * locations) = 0;
/**
@@ -224,9 +265,9 @@ class FileSystem {
* @param permissions Permissions for the new directory (negative value for the default permissions)
* @param createparent Create parent directories if they do not exist (may not be empty)
*/
- virtual void Mkdirs(const std::string & path, long permissions, bool createparent,
+ virtual void Mkdirs(const std::string & path, uint16_t permissions, bool createparent,
std::function<void(const Status &)> handler) = 0;
- virtual Status Mkdirs(const std::string & path, long permissions, bool createparent) = 0;
+ virtual Status Mkdirs(const std::string & path, uint16_t permissions, bool createparent) = 0;
/**
* Delete the given file or directory from the file system.
@@ -257,8 +298,8 @@ class FileSystem {
* @param permissions the bitmask to set it to (should be between 0 and 01777)
*/
virtual void SetPermission(const std::string & path,
- short permissions, const std::function<void(const Status &)> &handler) = 0;
- virtual Status SetPermission(const std::string & path, short permissions) = 0;
+ uint16_t permissions, const std::function<void(const Status &)> &handler) = 0;
+ virtual Status SetPermission(const std::string & path, uint16_t permissions) = 0;
/**
* Set Owner of a path (i.e. a file or a directory).
@@ -335,6 +376,8 @@ class FileSystem {
* @param callback The function to call when a reporting event occurs.
*/
virtual void SetFsEventCallback(fs_event_callback callback) = 0;
+
+ virtual Options get_options() = 0;
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a8edd4e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
index 1acfe1a..55093d0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
@@ -116,6 +116,13 @@ struct Options {
Authentication authentication;
static const Authentication kDefaultAuthentication = kSimple;
+ /**
+ * Block size in bytes.
+ * Default: 128 * 1024 * 1024 = 134217728
+ **/
+ long block_size;
+ static const long kDefaultBlockSize = 128*1024*1024;
+
Options();
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a8edd4e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
index 0187786..ffbb2bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
@@ -46,12 +46,16 @@ class Status {
static Status Canceled();
static Status PathNotFound(const char *msg);
static Status InvalidOffset(const char *msg);
+ static Status PathIsNotDirectory(const char *msg);
// success
bool ok() const { return code_ == 0; }
bool is_invalid_offset() const { return code_ == kInvalidOffset; }
+ // contains ENOENT error
+ bool pathNotFound() const { return code_ == kPathNotFound; }
+
// Returns the string "OK" for success.
std::string ToString() const;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a8edd4e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
index be57a7e..7fda4e2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
@@ -43,15 +43,18 @@ static constexpr tPort kDefaultPort = 8020;
/* Separate the handles used by the C api from the C++ API*/
struct hdfs_internal {
- hdfs_internal(FileSystem *p) : filesystem_(p) {}
+ hdfs_internal(FileSystem *p) : filesystem_(p), working_directory("/") {}
hdfs_internal(std::unique_ptr<FileSystem> p)
- : filesystem_(std::move(p)) {}
+ : filesystem_(std::move(p)), working_directory("/") {}
virtual ~hdfs_internal(){};
FileSystem *get_impl() { return filesystem_.get(); }
const FileSystem *get_impl() const { return filesystem_.get(); }
+ std::string get_working_directory() { return working_directory; }
+ void set_working_directory(std::string new_directory) { working_directory = new_directory; }
private:
std::unique_ptr<FileSystem> filesystem_;
+ std::string working_directory; //has to always start and end with '/'
};
struct hdfsFile_internal {
@@ -198,6 +201,7 @@ static int ReportCaughtNonException()
return Error(Status::Exception("Uncaught value not derived from std::exception", ""));
}
+/* return false on failure */
bool CheckSystem(hdfsFS fs) {
if (!fs) {
ReportError(ENODEV, "Cannot perform FS operations with null FS handle.");
@@ -208,27 +212,68 @@ bool CheckSystem(hdfsFS fs) {
}
/* return false on failure */
+bool CheckHandle(hdfsFile file) {
+ if (!file) {
+ ReportError(EBADF, "Cannot perform FS operations with null File handle.");
+ return false;
+ }
+ return true;
+}
+
+/* return false on failure */
bool CheckSystemAndHandle(hdfsFS fs, hdfsFile file) {
if (!CheckSystem(fs))
return false;
- if (!file) {
- ReportError(EBADF, "Cannot perform FS operations with null File handle.");
+ if (!CheckHandle(file))
return false;
- }
+
return true;
}
+optional<std::string> getAbsolutePath(hdfsFS fs, const char* path) {
+ //Does not support . (dot) and .. (double dot) semantics
+ if (!path || path[0] == '\0') {
+ Error(Status::InvalidArgument("getAbsolutePath: argument 'path' cannot be NULL or empty"));
+ return optional<std::string>();
+ }
+ if (path[0] != '/') {
+ //we know that working directory always ends with '/'
+ return fs->get_working_directory().append(path);
+ }
+ return optional<std::string>(path);
+}
+
/**
* C API implementations
**/
int hdfsFileIsOpenForRead(hdfsFile file) {
/* files can only be open for reads at the moment, do a quick check */
- if (file) {
- return 1; // Update implementation when we get file writing
+ if (!CheckHandle(file)){
+ return 0;
+ }
+ return 1; // Update implementation when we get file writing
+}
+
+int hdfsFileIsOpenForWrite(hdfsFile file) {
+ /* files can only be open for reads at the moment, so return false */
+ CheckHandle(file);
+ return -1; // Update implementation when we get file writing
+}
+
+int hdfsConfGetLong(const char *key, int64_t *val)
+{
+ try
+ {
+ errno = 0;
+ hdfsBuilder builder;
+ return hdfsBuilderConfGetLong(&builder, key, val);
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
}
- return 0;
}
hdfsFS doHdfsConnect(optional<std::string> nn, optional<tPort> port, optional<std::string> user, const Options & options) {
@@ -329,8 +374,12 @@ hdfsFile hdfsOpenFile(hdfsFS fs, const char *path, int flags, int bufferSize,
ReportError(ENODEV, "Cannot perform FS operations with null FS handle.");
return nullptr;
}
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return nullptr;
+ }
FileHandle *f = nullptr;
- Status stat = fs->get_impl()->Open(path, &f);
+ Status stat = fs->get_impl()->Open(*abs_path, &f);
if (!stat.ok()) {
Error(stat);
return nullptr;
@@ -364,11 +413,165 @@ int hdfsCloseFile(hdfsFS fs, hdfsFile file) {
}
}
+char* hdfsGetWorkingDirectory(hdfsFS fs, char *buffer, size_t bufferSize) {
+ try
+ {
+ errno = 0;
+ if (!CheckSystem(fs)) {
+ return nullptr;
+ }
+ std::string wd = fs->get_working_directory();
+ size_t size = wd.size();
+ if (size + 1 > bufferSize) {
+ std::stringstream ss;
+ ss << "hdfsGetWorkingDirectory: bufferSize is " << bufferSize <<
+ ", which is not enough to fit working directory of size " << (size + 1);
+ Error(Status::InvalidArgument(ss.str().c_str()));
+ return nullptr;
+ }
+ wd.copy(buffer, size);
+ buffer[size] = '\0';
+ return buffer;
+ } catch (const std::exception & e) {
+ ReportException(e);
+ return nullptr;
+ } catch (...) {
+ ReportCaughtNonException();
+ return nullptr;
+ }
+}
+
+int hdfsSetWorkingDirectory(hdfsFS fs, const char* path) {
+ try
+ {
+ errno = 0;
+ if (!CheckSystem(fs)) {
+ return -1;
+ }
+ optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
+ }
+ //Enforce last character to be '/'
+ std::string withSlash = *abs_path;
+ char last = withSlash.back();
+ if (last != '/'){
+ withSlash += '/';
+ }
+ fs->set_working_directory(withSlash);
+ return 0;
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
+}
+
+int hdfsAvailable(hdfsFS fs, hdfsFile file) {
+ //Since we do not have read ahead implemented, return 0 if fs and file are good;
+ errno = 0;
+ if (!CheckSystemAndHandle(fs, file)) {
+ return -1;
+ }
+ return 0;
+}
+
+tOffset hdfsGetDefaultBlockSize(hdfsFS fs) {
+ try {
+ errno = 0;
+ return fs->get_impl()->get_options().block_size;
+ } catch (const std::exception & e) {
+ ReportException(e);
+ return -1;
+ } catch (...) {
+ ReportCaughtNonException();
+ return -1;
+ }
+}
+
+tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path) {
+ try {
+ errno = 0;
+ if (!CheckSystem(fs)) {
+ return -1;
+ }
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
+ }
+ uint64_t block_size;
+ Status stat = fs->get_impl()->GetPreferredBlockSize(*abs_path, block_size);
+ if (!stat.ok()) {
+ if (stat.pathNotFound()){
+ return fs->get_impl()->get_options().block_size;
+ } else {
+ return Error(stat);
+ }
+ }
+ return block_size;
+ } catch (const std::exception & e) {
+ ReportException(e);
+ return -1;
+ } catch (...) {
+ ReportCaughtNonException();
+ return -1;
+ }
+}
+
+int hdfsSetReplication(hdfsFS fs, const char* path, int16_t replication) {
+ try {
+ errno = 0;
+ if (!CheckSystem(fs)) {
+ return -1;
+ }
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
+ }
+ if(replication < 1){
+ return Error(Status::InvalidArgument("SetReplication: argument 'replication' cannot be less than 1"));
+ }
+ Status stat;
+ stat = fs->get_impl()->SetReplication(*abs_path, replication);
+ if (!stat.ok()) {
+ return Error(stat);
+ }
+ return 0;
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
+}
+
+int hdfsUtime(hdfsFS fs, const char* path, tTime mtime, tTime atime) {
+ try {
+ errno = 0;
+ if (!CheckSystem(fs)) {
+ return -1;
+ }
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
+ }
+ Status stat;
+ stat = fs->get_impl()->SetTimes(*abs_path, mtime, atime);
+ if (!stat.ok()) {
+ return Error(stat);
+ }
+ return 0;
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
+}
+
tOffset hdfsGetCapacity(hdfsFS fs) {
try {
errno = 0;
if (!CheckSystem(fs)) {
- return -1;
+ return -1;
}
hdfs::FsInfo fs_info;
@@ -391,7 +594,7 @@ tOffset hdfsGetUsed(hdfsFS fs) {
try {
errno = 0;
if (!CheckSystem(fs)) {
- return -1;
+ return -1;
}
hdfs::FsInfo fs_info;
@@ -459,15 +662,41 @@ void StatInfoToHdfsFileInfo(hdfsFileInfo * file_info,
file_info->mLastAccess = stat_info.access_time;
}
+int hdfsExists(hdfsFS fs, const char *path) {
+ try {
+ errno = 0;
+ if (!CheckSystem(fs)) {
+ return -1;
+ }
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
+ }
+ hdfs::StatInfo stat_info;
+ Status stat = fs->get_impl()->GetFileInfo(*abs_path, stat_info);
+ if (!stat.ok()) {
+ return Error(stat);
+ }
+ return 0;
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
+}
+
hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char* path) {
try {
errno = 0;
if (!CheckSystem(fs)) {
return nullptr;
}
-
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return nullptr;
+ }
hdfs::StatInfo stat_info;
- Status stat = fs->get_impl()->GetFileInfo(path, stat_info);
+ Status stat = fs->get_impl()->GetFileInfo(*abs_path, stat_info);
if (!stat.ok()) {
Error(stat);
return nullptr;
@@ -491,9 +720,12 @@ hdfsFileInfo *hdfsListDirectory(hdfsFS fs, const char* path, int *numEntries) {
*numEntries = 0;
return nullptr;
}
-
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return nullptr;
+ }
std::shared_ptr<std::vector<StatInfo>> stat_infos;
- Status stat = fs->get_impl()->GetListing(path, stat_infos);
+ Status stat = fs->get_impl()->GetListing(*abs_path, stat_infos);
if (!stat.ok()) {
Error(stat);
*numEntries = 0;
@@ -540,12 +772,13 @@ int hdfsCreateDirectory(hdfsFS fs, const char* path) {
if (!CheckSystem(fs)) {
return -1;
}
- if (!path) {
- return Error(Status::InvalidArgument("hdfsCreateDirectory: argument 'path' cannot be NULL"));
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
}
Status stat;
- //-1 for default permissions and true for creating all non-existant parent directories
- stat = fs->get_impl()->Mkdirs(path, -1, true);
+ //Use default permissions and set true for creating all non-existant parent directories
+ stat = fs->get_impl()->Mkdirs(*abs_path, NameNodeOperations::GetDefaultPermissionMask(), true);
if (!stat.ok()) {
return Error(stat);
}
@@ -563,11 +796,12 @@ int hdfsDelete(hdfsFS fs, const char* path, int recursive) {
if (!CheckSystem(fs)) {
return -1;
}
- if (!path) {
- return Error(Status::InvalidArgument("hdfsDelete: argument 'path' cannot be NULL"));
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
}
Status stat;
- stat = fs->get_impl()->Delete(path, recursive);
+ stat = fs->get_impl()->Delete(*abs_path, recursive);
if (!stat.ok()) {
return Error(stat);
}
@@ -585,14 +819,13 @@ int hdfsRename(hdfsFS fs, const char* oldPath, const char* newPath) {
if (!CheckSystem(fs)) {
return -1;
}
- if (!oldPath) {
- return Error(Status::InvalidArgument("hdfsRename: argument 'oldPath' cannot be NULL"));
- }
- if (!newPath) {
- return Error(Status::InvalidArgument("hdfsRename: argument 'newPath' cannot be NULL"));
+ const optional<std::string> old_abs_path = getAbsolutePath(fs, oldPath);
+ const optional<std::string> new_abs_path = getAbsolutePath(fs, newPath);
+ if(!old_abs_path || !new_abs_path) {
+ return -1;
}
Status stat;
- stat = fs->get_impl()->Rename(oldPath, newPath);
+ stat = fs->get_impl()->Rename(*old_abs_path, *new_abs_path);
if (!stat.ok()) {
return Error(stat);
}
@@ -610,14 +843,15 @@ int hdfsChmod(hdfsFS fs, const char* path, short mode){
if (!CheckSystem(fs)) {
return -1;
}
- if (!path) {
- return Error(Status::InvalidArgument("hdfsChmod: argument 'path' cannot be NULL"));
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
}
Status stat = NameNodeOperations::CheckValidPermissionMask(mode);
if (!stat.ok()) {
return Error(stat);
}
- stat = fs->get_impl()->SetPermission(path, mode);
+ stat = fs->get_impl()->SetPermission(*abs_path, mode);
if (!stat.ok()) {
return Error(stat);
}
@@ -635,14 +869,15 @@ int hdfsChown(hdfsFS fs, const char* path, const char *owner, const char *group)
if (!CheckSystem(fs)) {
return -1;
}
- if (!path) {
- return Error(Status::InvalidArgument("hdfsChown: argument 'path' cannot be NULL"));
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
}
std::string own = (owner) ? owner : "";
std::string grp = (group) ? group : "";
Status stat;
- stat = fs->get_impl()->SetOwner(path, own, grp);
+ stat = fs->get_impl()->SetOwner(*abs_path, own, grp);
if (!stat.ok()) {
return Error(stat);
}
@@ -660,14 +895,15 @@ int hdfsCreateSnapshot(hdfsFS fs, const char* path, const char* name) {
if (!CheckSystem(fs)) {
return -1;
}
- if (!path) {
- return Error(Status::InvalidArgument("hdfsCreateSnapshot: argument 'path' cannot be NULL"));
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
}
Status stat;
if(!name){
- stat = fs->get_impl()->CreateSnapshot(path, "");
+ stat = fs->get_impl()->CreateSnapshot(*abs_path, "");
} else {
- stat = fs->get_impl()->CreateSnapshot(path, name);
+ stat = fs->get_impl()->CreateSnapshot(*abs_path, name);
}
if (!stat.ok()) {
return Error(stat);
@@ -686,14 +922,15 @@ int hdfsDeleteSnapshot(hdfsFS fs, const char* path, const char* name) {
if (!CheckSystem(fs)) {
return -1;
}
- if (!path) {
- return Error(Status::InvalidArgument("hdfsDeleteSnapshot: argument 'path' cannot be NULL"));
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
}
if (!name) {
return Error(Status::InvalidArgument("hdfsDeleteSnapshot: argument 'name' cannot be NULL"));
}
Status stat;
- stat = fs->get_impl()->DeleteSnapshot(path, name);
+ stat = fs->get_impl()->DeleteSnapshot(*abs_path, name);
if (!stat.ok()) {
return Error(stat);
}
@@ -711,11 +948,12 @@ int hdfsAllowSnapshot(hdfsFS fs, const char* path) {
if (!CheckSystem(fs)) {
return -1;
}
- if (!path) {
- return Error(Status::InvalidArgument("hdfsAllowSnapshot: argument 'path' cannot be NULL"));
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
}
Status stat;
- stat = fs->get_impl()->AllowSnapshot(path);
+ stat = fs->get_impl()->AllowSnapshot(*abs_path);
if (!stat.ok()) {
return Error(stat);
}
@@ -733,11 +971,12 @@ int hdfsDisallowSnapshot(hdfsFS fs, const char* path) {
if (!CheckSystem(fs)) {
return -1;
}
- if (!path) {
- return Error(Status::InvalidArgument("hdfsDisallowSnapshot: argument 'path' cannot be NULL"));
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
}
Status stat;
- stat = fs->get_impl()->DisallowSnapshot(path);
+ stat = fs->get_impl()->DisallowSnapshot(*abs_path);
if (!stat.ok()) {
return Error(stat);
}
@@ -793,6 +1032,55 @@ tSize hdfsRead(hdfsFS fs, hdfsFile file, void *buffer, tSize length) {
}
}
+int hdfsUnbufferFile(hdfsFile file) {
+ //Currently we are not doing any buffering
+ CheckHandle(file);
+ return -1;
+}
+
+int hdfsFileGetReadStatistics(hdfsFile file, struct hdfsReadStatistics **stats) {
+ try
+ {
+ errno = 0;
+ if (!CheckHandle(file)) {
+ return -1;
+ }
+ *stats = new hdfsReadStatistics;
+ memset(*stats, 0, sizeof(hdfsReadStatistics));
+ (*stats)->totalBytesRead = file->get_impl()->get_bytes_read();
+ return 0;
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
+}
+
+int hdfsFileClearReadStatistics(hdfsFile file) {
+ try
+ {
+ errno = 0;
+ if (!CheckHandle(file)) {
+ return -1;
+ }
+ file->get_impl()->clear_bytes_read();
+ return 0;
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
+}
+
+int64_t hdfsReadStatisticsGetRemoteBytesRead(const struct hdfsReadStatistics *stats) {
+ return stats->totalBytesRead - stats->totalLocalBytesRead;
+}
+
+void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats) {
+ errno = 0;
+ delete stats;
+}
+
/* 0 on success, -1 on error*/
int hdfsSeek(hdfsFS fs, hdfsFile file, tOffset desiredPos) {
try
@@ -868,9 +1156,12 @@ int hdfsGetBlockLocations(hdfsFS fs, const char *path, struct hdfsBlockLocations
ReportError(EINVAL, "Null pointer passed to hdfsGetBlockLocations");
return -1;
}
-
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
+ }
std::shared_ptr<FileBlockLocation> ppLocations;
- Status stat = fs->get_impl()->GetBlockLocations(path, &ppLocations);
+ Status stat = fs->get_impl()->GetBlockLocations(*abs_path, 0, std::numeric_limits<int64_t>::max(), &ppLocations);
if (!stat.ok()) {
return Error(stat);
}
@@ -943,6 +1234,59 @@ int hdfsFreeBlockLocations(struct hdfsBlockLocations * blockLocations) {
return 0;
}
+char*** hdfsGetHosts(hdfsFS fs, const char* path, tOffset start, tOffset length) {
+ try
+ {
+ errno = 0;
+ if (!CheckSystem(fs)) {
+ return nullptr;
+ }
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return nullptr;
+ }
+ std::shared_ptr<FileBlockLocation> ppLocations;
+ Status stat = fs->get_impl()->GetBlockLocations(*abs_path, start, length, &ppLocations);
+ if (!stat.ok()) {
+ Error(stat);
+ return nullptr;
+ }
+ const std::vector<BlockLocation> & ppBlockLocations = ppLocations->getBlockLocations();
+ char ***hosts = new char**[ppBlockLocations.size() + 1];
+ for (size_t i=0; i < ppBlockLocations.size(); i++) {
+ const std::vector<DNInfo> & ppDNInfos = ppBlockLocations[i].getDataNodes();
+ hosts[i] = new char*[ppDNInfos.size() + 1];
+ for (size_t j=0; j < ppDNInfos.size(); j++) {
+ auto ppDNInfo = ppDNInfos[j];
+ hosts[i][j] = new char[ppDNInfo.getHostname().size() + 1];
+ strncpy(hosts[i][j], ppDNInfo.getHostname().c_str(), ppDNInfo.getHostname().size() + 1);
+ }
+ hosts[i][ppDNInfos.size()] = nullptr;
+ }
+ hosts[ppBlockLocations.size()] = nullptr;
+ return hosts;
+ } catch (const std::exception & e) {
+ ReportException(e);
+ return nullptr;
+ } catch (...) {
+ ReportCaughtNonException();
+ return nullptr;
+ }
+}
+
+void hdfsFreeHosts(char ***blockHosts) {
+ errno = 0;
+ if (blockHosts == nullptr)
+ return;
+
+ for (size_t i = 0; blockHosts[i]; i++) {
+ for (size_t j = 0; blockHosts[i][j]; j++) {
+ delete[] blockHosts[i][j];
+ }
+ delete[] blockHosts[i];
+ }
+ delete blockHosts;
+}
/*******************************************************************
* EVENT CALLBACKS
@@ -1234,6 +1578,28 @@ int hdfsBuilderConfGetInt(struct hdfsBuilder *bld, const char *key, int32_t *val
}
}
+int hdfsBuilderConfGetLong(struct hdfsBuilder *bld, const char *key, int64_t *val)
+{
+ try
+ {
+ errno = 0;
+ // Pull from default configuration
+ optional<int64_t> value = bld->config.GetInt(key);
+ if (value)
+ {
+ *val = *value;
+ return 0;
+ }
+ // If not found, don't change val
+ ReportError(EINVAL, "Could not get Builder value");
+ return 0;
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
+}
+
/**
* Logging functions
**/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a8edd4e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc
index 70775b8..6778bad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc
@@ -144,6 +144,7 @@ Options HdfsConfiguration::GetOptions() {
OptionalSet(result.max_rpc_retries, GetInt(kIpcClientConnectMaxRetriesKey));
OptionalSet(result.rpc_retry_delay_ms, GetInt(kIpcClientConnectRetryIntervalKey));
OptionalSet(result.defaultFS, GetUri(kFsDefaultFsKey));
+ OptionalSet(result.block_size, GetInt(kDfsBlockSizeKey));
OptionalSet(result.failover_max_retries, GetInt(kDfsClientFailoverMaxAttempts));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a8edd4e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.h
index 459364f..d6f902e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.h
@@ -45,6 +45,7 @@ class HdfsConfiguration : public Configuration {
static constexpr const char * kHadoopSecurityAuthenticationKey = "hadoop.security.authentication";
static constexpr const char * kHadoopSecurityAuthentication_simple = "simple";
static constexpr const char * kHadoopSecurityAuthentication_kerberos = "kerberos";
+ static constexpr const char * kDfsBlockSizeKey = "dfs.blocksize";
static constexpr const char * kDfsClientFailoverMaxAttempts = "dfs.client.failover.max.attempts";
static constexpr const char * kDfsClientFailoverConnectionRetriesOnTimeouts = "dfs.client.failover.connection.retries.on.timeouts";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a8edd4e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc
index 29b45b2..a889be5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc
@@ -28,6 +28,7 @@ const int Options::kDefaultRpcRetryDelayMs;
const unsigned int Options::kDefaultHostExclusionDuration;
const unsigned int Options::kDefaultFailoverMaxRetries;
const unsigned int Options::kDefaultFailoverConnectionMaxRetries;
+const long Options::kDefaultBlockSize;
Options::Options() : rpc_timeout(kDefaultRpcTimeout),
rpc_connect_timeout(kDefaultRpcConnectTimeout),
@@ -37,7 +38,8 @@ Options::Options() : rpc_timeout(kDefaultRpcTimeout),
defaultFS(),
failover_max_retries(kDefaultFailoverMaxRetries),
failover_connection_max_retries(kDefaultFailoverConnectionMaxRetries),
- authentication(kDefaultAuthentication) {}
+ authentication(kDefaultAuthentication),
+ block_size(kDefaultBlockSize) {}
std::string NamenodeInfo::get_host() const {
return uri.get_host();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a8edd4e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
index b351900..540f8c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
@@ -89,6 +89,10 @@ Status Status::ResourceUnavailable(const char *msg) {
return Status(kResourceUnavailable, msg);
}
+Status Status::PathIsNotDirectory(const char *msg) {
+ return Status(kNotADirectory, msg);
+}
+
Status Status::Unimplemented() {
return Status(kUnimplemented, "");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a8edd4e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
index 7bb1e30..853ada8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
@@ -107,6 +107,15 @@ std::string SafeDisconnect(asio::ip::tcp::socket *sock) {
return err;
}
+bool IsHighBitSet(uint64_t num) {
+ uint64_t firstBit = (uint64_t) 1 << 63;
+ if (num & firstBit) {
+ return true;
+ } else {
+ return false;
+ }
+}
+
}
void ShutdownProtobufLibrary_C() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a8edd4e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.h
index da63435..7f0e572 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.h
@@ -110,6 +110,8 @@ inline asio::ip::tcp::socket *get_asio_socket_ptr<asio::ip::tcp::socket>
return s;
}
+//Check if the high bit is set
+bool IsHighBitSet(uint64_t num);
}
#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a8edd4e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
index f40b81c..2600944 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
@@ -41,7 +41,7 @@ FileHandleImpl::FileHandleImpl(const std::string & cluster_name,
std::shared_ptr<BadDataNodeTracker> bad_data_nodes,
std::shared_ptr<LibhdfsEvents> event_handlers)
: cluster_name_(cluster_name), path_(path), io_service_(io_service), client_name_(client_name), file_info_(file_info),
- bad_node_tracker_(bad_data_nodes), offset_(0), cancel_state_(CancelTracker::New()), event_handlers_(event_handlers) {
+ bad_node_tracker_(bad_data_nodes), offset_(0), cancel_state_(CancelTracker::New()), event_handlers_(event_handlers), bytes_read_(0) {
LOG_TRACE(kFileHandle, << "FileHandleImpl::FileHandleImpl("
<< FMT_THIS_ADDR << ", ...) called");
@@ -68,6 +68,7 @@ void FileHandleImpl::PositionRead(
bad_node_tracker_->AddBadNode(contacted_datanode);
}
+ bytes_read_ += bytes_read;
handler(status, bytes_read);
};
@@ -352,4 +353,8 @@ bool FileHandle::ShouldExclude(const Status &s) {
}
}
+uint64_t FileHandleImpl::get_bytes_read() { return bytes_read_; }
+
+void FileHandleImpl::clear_bytes_read() { bytes_read_ = 0; }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a8edd4e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
index 38c1fec..03c55ff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
@@ -113,6 +113,12 @@ public:
**/
std::shared_ptr<LibhdfsEvents> get_event_handlers();
+ /* how many bytes have been successfully read */
+ virtual uint64_t get_bytes_read() override;
+
+ /* resets the number of bytes read to zero */
+ virtual void clear_bytes_read() override;
+
protected:
virtual std::shared_ptr<BlockReader> CreateBlockReader(const BlockReaderOptions &options,
std::shared_ptr<DataNodeConnection> dn,
@@ -133,6 +139,7 @@ private:
CancelHandle cancel_state_;
ReaderGroup readers_;
std::shared_ptr<LibhdfsEvents> event_handlers_;
+ uint64_t bytes_read_;
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a8edd4e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
index ae55e0b..de6ebb7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
@@ -243,14 +243,13 @@ void FileSystemImpl::Open(
<< FMT_THIS_ADDR << ", path="
<< path << ") called");
- nn_.GetBlockLocations(path, [this, path, handler](const Status &stat, std::shared_ptr<const struct FileInfo> file_info) {
+ nn_.GetBlockLocations(path, 0, std::numeric_limits<int64_t>::max(), [this, path, handler](const Status &stat, std::shared_ptr<const struct FileInfo> file_info) {
if(!stat.ok()) {
LOG_INFO(kFileSystem, << "FileSystemImpl::Open failed to get block locations. status=" << stat.ToString());
if(stat.get_server_exception_type() == Status::kStandbyException) {
LOG_INFO(kFileSystem, << "Operation not allowed on standby datanode");
}
}
-
handler(stat, stat.ok() ? new FileHandleImpl(cluster_name_, path, &io_service_->io_service(), client_name_, file_info, bad_node_tracker_, event_handlers_)
: nullptr);
});
@@ -326,13 +325,24 @@ BlockLocation LocatedBlockToBlockLocation(const hadoop::hdfs::LocatedBlockProto
return result;
}
-void FileSystemImpl::GetBlockLocations(const std::string & path,
+void FileSystemImpl::GetBlockLocations(const std::string & path, uint64_t offset, uint64_t length,
const std::function<void(const Status &, std::shared_ptr<FileBlockLocation> locations)> handler)
{
LOG_DEBUG(kFileSystem, << "FileSystemImpl::GetBlockLocations("
<< FMT_THIS_ADDR << ", path="
<< path << ") called");
+ //Protobuf gives an error 'Negative value is not supported'
+ //if the high bit is set in uint64 in GetBlockLocations
+ if (IsHighBitSet(offset)) {
+ handler(Status::InvalidArgument("GetBlockLocations: argument 'offset' cannot have high bit set"), nullptr);
+ return;
+ }
+ if (IsHighBitSet(length)) {
+ handler(Status::InvalidArgument("GetBlockLocations: argument 'length' cannot have high bit set"), nullptr);
+ return;
+ }
+
auto conversion = [handler](const Status & status, std::shared_ptr<const struct FileInfo> fileInfo) {
if (status.ok()) {
auto result = std::make_shared<FileBlockLocation>();
@@ -354,10 +364,10 @@ void FileSystemImpl::GetBlockLocations(const std::string & path,
}
};
- nn_.GetBlockLocations(path, conversion);
+ nn_.GetBlockLocations(path, offset, length, conversion);
}
-Status FileSystemImpl::GetBlockLocations(const std::string & path,
+Status FileSystemImpl::GetBlockLocations(const std::string & path, uint64_t offset, uint64_t length,
std::shared_ptr<FileBlockLocation> * fileBlockLocations)
{
LOG_DEBUG(kFileSystem, << "FileSystemImpl::[sync]GetBlockLocations("
@@ -375,7 +385,7 @@ Status FileSystemImpl::GetBlockLocations(const std::string & path,
callstate->set_value(std::make_tuple(s,blockInfo));
};
- GetBlockLocations(path, callback);
+ GetBlockLocations(path, offset, length, callback);
/* wait for async to finish */
auto returnstate = future.get();
@@ -390,6 +400,119 @@ Status FileSystemImpl::GetBlockLocations(const std::string & path,
return stat;
}
+void FileSystemImpl::GetPreferredBlockSize(const std::string &path,
+ const std::function<void(const Status &, const uint64_t &)> &handler) {
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl::GetPreferredBlockSize("
+ << FMT_THIS_ADDR << ", path="
+ << path << ") called");
+
+ nn_.GetPreferredBlockSize(path, handler);
+}
+
+Status FileSystemImpl::GetPreferredBlockSize(const std::string &path, uint64_t & block_size) {
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl::[sync]GetPreferredBlockSize("
+ << FMT_THIS_ADDR << ", path="
+ << path << ") called");
+
+ auto callstate = std::make_shared<std::promise<std::tuple<Status, uint64_t>>>();
+ std::future<std::tuple<Status, uint64_t>> future(callstate->get_future());
+
+ /* wrap async FileSystem::GetPreferredBlockSize with promise to make it a blocking call */
+ auto h = [callstate](const Status &s, const uint64_t & bsize) {
+ callstate->set_value(std::make_tuple(s, bsize));
+ };
+
+ GetPreferredBlockSize(path, h);
+
+ /* block until promise is set */
+ auto returnstate = future.get();
+ Status stat = std::get<0>(returnstate);
+ uint64_t size = std::get<1>(returnstate);
+
+ if (!stat.ok()) {
+ return stat;
+ }
+
+ block_size = size;
+ return stat;
+}
+
+void FileSystemImpl::SetReplication(const std::string & path, int16_t replication, std::function<void(const Status &)> handler) {
+ LOG_DEBUG(kFileSystem,
+ << "FileSystemImpl::SetReplication(" << FMT_THIS_ADDR << ", path=" << path <<
+ ", replication=" << replication << ") called");
+
+ if (path.empty()) {
+ handler(Status::InvalidArgument("SetReplication: argument 'path' cannot be empty"));
+ return;
+ }
+ Status replStatus = NameNodeOperations::CheckValidReplication(replication);
+ if (!replStatus.ok()) {
+ handler(replStatus);
+ return;
+ }
+
+ nn_.SetReplication(path, replication, handler);
+}
+
+Status FileSystemImpl::SetReplication(const std::string & path, int16_t replication) {
+ LOG_DEBUG(kFileSystem,
+ << "FileSystemImpl::[sync]SetReplication(" << FMT_THIS_ADDR << ", path=" << path <<
+ ", replication=" << replication << ") called");
+
+ auto callstate = std::make_shared<std::promise<Status>>();
+ std::future<Status> future(callstate->get_future());
+
+ /* wrap async FileSystem::SetReplication with promise to make it a blocking call */
+ auto h = [callstate](const Status &s) {
+ callstate->set_value(s);
+ };
+
+ SetReplication(path, replication, h);
+
+ /* block until promise is set */
+ auto returnstate = future.get();
+ Status stat = returnstate;
+
+ return stat;
+}
+
+void FileSystemImpl::SetTimes(const std::string & path, uint64_t mtime, uint64_t atime,
+ std::function<void(const Status &)> handler) {
+ LOG_DEBUG(kFileSystem,
+ << "FileSystemImpl::SetTimes(" << FMT_THIS_ADDR << ", path=" << path <<
+ ", mtime=" << mtime << ", atime=" << atime << ") called");
+
+ if (path.empty()) {
+ handler(Status::InvalidArgument("SetTimes: argument 'path' cannot be empty"));
+ return;
+ }
+
+ nn_.SetTimes(path, mtime, atime, handler);
+}
+
+Status FileSystemImpl::SetTimes(const std::string & path, uint64_t mtime, uint64_t atime) {
+ LOG_DEBUG(kFileSystem,
+ << "FileSystemImpl::[sync]SetTimes(" << FMT_THIS_ADDR << ", path=" << path <<
+ ", mtime=" << mtime << ", atime=" << atime << ") called");
+
+ auto callstate = std::make_shared<std::promise<Status>>();
+ std::future<Status> future(callstate->get_future());
+
+ /* wrap async FileSystem::SetTimes with promise to make it a blocking call */
+ auto h = [callstate](const Status &s) {
+ callstate->set_value(s);
+ };
+
+ SetTimes(path, mtime, atime, h);
+
+ /* block until promise is set */
+ auto returnstate = future.get();
+ Status stat = returnstate;
+
+ return stat;
+}
+
void FileSystemImpl::GetFileInfo(
const std::string &path,
const std::function<void(const Status &, const StatInfo &)> &handler) {
@@ -543,7 +666,7 @@ Status FileSystemImpl::GetListing(const std::string &path, std::shared_ptr<std::
return stat;
}
-void FileSystemImpl::Mkdirs(const std::string & path, long permissions, bool createparent,
+void FileSystemImpl::Mkdirs(const std::string & path, uint16_t permissions, bool createparent,
std::function<void(const Status &)> handler) {
LOG_DEBUG(kFileSystem,
<< "FileSystemImpl::Mkdirs(" << FMT_THIS_ADDR << ", path=" << path <<
@@ -554,10 +677,16 @@ void FileSystemImpl::Mkdirs(const std::string & path, long permissions, bool cre
return;
}
+ Status permStatus = NameNodeOperations::CheckValidPermissionMask(permissions);
+ if (!permStatus.ok()) {
+ handler(permStatus);
+ return;
+ }
+
nn_.Mkdirs(path, permissions, createparent, handler);
}
-Status FileSystemImpl::Mkdirs(const std::string & path, long permissions, bool createparent) {
+Status FileSystemImpl::Mkdirs(const std::string & path, uint16_t permissions, bool createparent) {
LOG_DEBUG(kFileSystem,
<< "FileSystemImpl::[sync]Mkdirs(" << FMT_THIS_ADDR << ", path=" << path <<
", permissions=" << permissions << ", createparent=" << createparent << ") called");
@@ -653,7 +782,7 @@ Status FileSystemImpl::Rename(const std::string &oldPath, const std::string &new
}
void FileSystemImpl::SetPermission(const std::string & path,
- short permissions, const std::function<void(const Status &)> &handler) {
+ uint16_t permissions, const std::function<void(const Status &)> &handler) {
LOG_DEBUG(kFileSystem,
<< "FileSystemImpl::SetPermission(" << FMT_THIS_ADDR << ", path=" << path << ", permissions=" << permissions << ") called");
@@ -670,7 +799,7 @@ void FileSystemImpl::SetPermission(const std::string & path,
nn_.SetPermission(path, permissions, handler);
}
-Status FileSystemImpl::SetPermission(const std::string & path, short permissions) {
+Status FileSystemImpl::SetPermission(const std::string & path, uint16_t permissions) {
LOG_DEBUG(kFileSystem,
<< "FileSystemImpl::[sync]SetPermission(" << FMT_THIS_ADDR << ", path=" << path << ", permissions=" << permissions << ") called");
@@ -896,4 +1025,8 @@ std::shared_ptr<LibhdfsEvents> FileSystemImpl::get_event_handlers() {
return event_handlers_;
}
+Options FileSystemImpl::get_options() {
+ return options_;
+}
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a8edd4e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
index 73ad7eb..75a43f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
@@ -65,6 +65,16 @@ public:
&handler) override;
Status Open(const std::string &path, FileHandle **handle) override;
+ virtual void GetPreferredBlockSize(const std::string &path,
+ const std::function<void(const Status &, const uint64_t &)> &handler) override;
+ virtual Status GetPreferredBlockSize(const std::string &path, uint64_t & block_size) override;
+
+ virtual void SetReplication(const std::string & path, int16_t replication, std::function<void(const Status &)> handler) override;
+ virtual Status SetReplication(const std::string & path, int16_t replication) override;
+
+ void SetTimes(const std::string & path, uint64_t mtime, uint64_t atime, std::function<void(const Status &)> handler) override;
+ Status SetTimes(const std::string & path, uint64_t mtime, uint64_t atime) override;
+
void GetFileInfo(
const std::string &path,
const std::function<void(const Status &, const StatInfo &)> &handler) override;
@@ -88,14 +98,14 @@ public:
Status GetListing(const std::string &path, std::shared_ptr<std::vector<StatInfo>> &stat_infos) override;
- virtual void GetBlockLocations(const std::string & path,
+ virtual void GetBlockLocations(const std::string & path, uint64_t offset, uint64_t length,
const std::function<void(const Status &, std::shared_ptr<FileBlockLocation> locations)> ) override;
- virtual Status GetBlockLocations(const std::string & path,
+ virtual Status GetBlockLocations(const std::string & path, uint64_t offset, uint64_t length,
std::shared_ptr<FileBlockLocation> * locations) override;
- virtual void Mkdirs(const std::string & path, long permissions, bool createparent,
+ virtual void Mkdirs(const std::string & path, uint16_t permissions, bool createparent,
std::function<void(const Status &)> handler) override;
- virtual Status Mkdirs(const std::string & path, long permissions, bool createparent) override;
+ virtual Status Mkdirs(const std::string & path, uint16_t permissions, bool createparent) override;
virtual void Delete(const std::string &path, bool recursive,
const std::function<void(const Status &)> &handler) override;
@@ -106,8 +116,8 @@ public:
virtual Status Rename(const std::string &oldPath, const std::string &newPath) override;
virtual void SetPermission(const std::string & path,
- short permissions, const std::function<void(const Status &)> &handler) override;
- virtual Status SetPermission(const std::string & path, short permissions) override;
+ uint16_t permissions, const std::function<void(const Status &)> &handler) override;
+ virtual Status SetPermission(const std::string & path, uint16_t permissions) override;
virtual void SetOwner(const std::string & path, const std::string & username,
const std::string & groupname, const std::function<void(const Status &)> &handler) override;
@@ -166,6 +176,8 @@ public:
/* all monitored events will need to lookup handlers */
std::shared_ptr<LibhdfsEvents> get_event_handlers();
+ Options get_options();
+
private:
const Options options_;
const std::string client_name_;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a8edd4e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
index 8b4c126..27ccb5d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
@@ -39,14 +39,26 @@ namespace hdfs {
* NAMENODE OPERATIONS
****************************************************************************/
-Status NameNodeOperations::CheckValidPermissionMask(short permissions) {
- if (permissions < 0 || permissions > 01777) {
+uint16_t NameNodeOperations::GetDefaultPermissionMask() {
+ return 0755;
+}
+
+Status NameNodeOperations::CheckValidPermissionMask(uint16_t permissions) {
+ if (permissions > 01777) {
std::stringstream errormsg;
- errormsg << "IsValidPermissionMask: argument 'permissions' is " << std::oct
+ errormsg << "CheckValidPermissionMask: argument 'permissions' is " << std::oct
<< std::showbase << permissions << " (should be between 0 and 01777)";
- //Avoid copying by binding errormsg.str() to a const reference, which extends its lifetime
- const std::string& tmp = errormsg.str();
- return Status::InvalidArgument(tmp.c_str());
+ return Status::InvalidArgument(errormsg.str().c_str());
+ }
+ return Status::OK();
+}
+
+Status NameNodeOperations::CheckValidReplication(uint16_t replication) {
+ if (replication < 1 || replication > 512) {
+ std::stringstream errormsg;
+ errormsg << "CheckValidReplication: argument 'replication' is "
+ << replication << " (should be between 1 and 512)";
+ return Status::InvalidArgument(errormsg.str().c_str());
}
return Status::OK();
}
@@ -57,7 +69,7 @@ void NameNodeOperations::Connect(const std::string &cluster_name,
engine_.Connect(cluster_name, servers, handler);
}
-void NameNodeOperations::GetBlockLocations(const std::string & path,
+void NameNodeOperations::GetBlockLocations(const std::string & path, uint64_t offset, uint64_t length,
std::function<void(const Status &, std::shared_ptr<const struct FileInfo>)> handler)
{
using ::hadoop::hdfs::GetBlockLocationsRequestProto;
@@ -71,10 +83,21 @@ void NameNodeOperations::GetBlockLocations(const std::string & path,
return;
}
+ //Protobuf gives an error 'Negative value is not supported'
+ //if the high bit is set in uint64 in GetBlockLocations
+ if (IsHighBitSet(offset)) {
+ handler(Status::InvalidArgument("GetBlockLocations: argument 'offset' cannot have high bit set"), nullptr);
+ return;
+ }
+ if (IsHighBitSet(length)) {
+ handler(Status::InvalidArgument("GetBlockLocations: argument 'length' cannot have high bit set"), nullptr);
+ return;
+ }
+
GetBlockLocationsRequestProto req;
req.set_src(path);
- req.set_offset(0);
- req.set_length(std::numeric_limits<long long>::max());
+ req.set_offset(offset);
+ req.set_length(length);
auto resp = std::make_shared<GetBlockLocationsResponseProto>();
@@ -104,6 +127,106 @@ void NameNodeOperations::GetBlockLocations(const std::string & path,
});
}
+void NameNodeOperations::GetPreferredBlockSize(const std::string & path,
+ std::function<void(const Status &, const uint64_t)> handler)
+{
+ using ::hadoop::hdfs::GetPreferredBlockSizeRequestProto;
+ using ::hadoop::hdfs::GetPreferredBlockSizeResponseProto;
+
+ LOG_TRACE(kFileSystem, << "NameNodeOperations::GetPreferredBlockSize("
+ << FMT_THIS_ADDR << ", path=" << path << ") called");
+
+ if (path.empty()) {
+ handler(Status::InvalidArgument("GetPreferredBlockSize: argument 'path' cannot be empty"), -1);
+ return;
+ }
+
+ GetPreferredBlockSizeRequestProto req;
+ req.set_filename(path);
+
+ auto resp = std::make_shared<GetPreferredBlockSizeResponseProto>();
+
+ namenode_.GetPreferredBlockSize(&req, resp, [resp, handler, path](const Status &stat) {
+ if (stat.ok() && resp -> has_bsize()) {
+ uint64_t block_size = resp -> bsize();
+ handler(stat, block_size);
+ } else {
+ handler(stat, -1);
+ }
+ });
+}
+
+void NameNodeOperations::SetReplication(const std::string & path, int16_t replication,
+ std::function<void(const Status &)> handler)
+{
+ using ::hadoop::hdfs::SetReplicationRequestProto;
+ using ::hadoop::hdfs::SetReplicationResponseProto;
+
+ LOG_TRACE(kFileSystem,
+ << "NameNodeOperations::SetReplication(" << FMT_THIS_ADDR << ", path=" << path <<
+ ", replication=" << replication << ") called");
+
+ if (path.empty()) {
+ handler(Status::InvalidArgument("SetReplication: argument 'path' cannot be empty"));
+ return;
+ }
+ Status replStatus = CheckValidReplication(replication);
+ if (!replStatus.ok()) {
+ handler(replStatus);
+ return;
+ }
+ SetReplicationRequestProto req;
+ req.set_src(path);
+ req.set_replication(replication);
+
+ auto resp = std::make_shared<SetReplicationResponseProto>();
+
+ namenode_.SetReplication(&req, resp, [resp, handler, path](const Status &stat) {
+ if (stat.ok()) {
+ // Checking resp
+ if(resp -> has_result() && resp ->result() == 1) {
+ handler(stat);
+ } else {
+ //NameNode does not specify why there is no result, in my testing it was happening when the path is not found
+ std::string errormsg = "No such file or directory: " + path;
+ Status statNew = Status::PathNotFound(errormsg.c_str());
+ handler(statNew);
+ }
+ } else {
+ handler(stat);
+ }
+ });
+}
+
+void NameNodeOperations::SetTimes(const std::string & path, uint64_t mtime, uint64_t atime,
+ std::function<void(const Status &)> handler)
+{
+ using ::hadoop::hdfs::SetTimesRequestProto;
+ using ::hadoop::hdfs::SetTimesResponseProto;
+
+ LOG_TRACE(kFileSystem,
+ << "NameNodeOperations::SetTimes(" << FMT_THIS_ADDR << ", path=" << path <<
+ ", mtime=" << mtime << ", atime=" << atime << ") called");
+
+ if (path.empty()) {
+ handler(Status::InvalidArgument("SetTimes: argument 'path' cannot be empty"));
+ return;
+ }
+
+ SetTimesRequestProto req;
+ req.set_src(path);
+ req.set_mtime(mtime);
+ req.set_atime(atime);
+
+ auto resp = std::make_shared<SetTimesResponseProto>();
+
+ namenode_.SetTimes(&req, resp, [resp, handler, path](const Status &stat) {
+ handler(stat);
+ });
+}
+
+
+
void NameNodeOperations::GetFileInfo(const std::string & path,
std::function<void(const Status &, const StatInfo &)> handler)
{
@@ -216,7 +339,7 @@ void NameNodeOperations::GetListing(
});
}
-void NameNodeOperations::Mkdirs(const std::string & path, long permissions, bool createparent,
+void NameNodeOperations::Mkdirs(const std::string & path, uint16_t permissions, bool createparent,
std::function<void(const Status &)> handler)
{
using ::hadoop::hdfs::MkdirsRequestProto;
@@ -232,13 +355,14 @@ void NameNodeOperations::Mkdirs(const std::string & path, long permissions, bool
}
MkdirsRequestProto req;
+ Status permStatus = CheckValidPermissionMask(permissions);
+ if (!permStatus.ok()) {
+ handler(permStatus);
+ return;
+ }
req.set_src(path);
hadoop::hdfs::FsPermissionProto *perm = req.mutable_masked();
- if (permissions < 0) {
- perm->set_perm(0755);
- } else {
- perm->set_perm(permissions);
- }
+ perm->set_perm(permissions);
req.set_createparent(createparent);
auto resp = std::make_shared<MkdirsResponseProto>();
@@ -336,7 +460,7 @@ void NameNodeOperations::Rename(const std::string & oldPath, const std::string &
}
void NameNodeOperations::SetPermission(const std::string & path,
- short permissions, std::function<void(const Status &)> handler) {
+ uint16_t permissions, std::function<void(const Status &)> handler) {
using ::hadoop::hdfs::SetPermissionRequestProto;
using ::hadoop::hdfs::SetPermissionResponseProto;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a8edd4e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
index 9651570..3afa2e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
@@ -48,15 +48,28 @@ public:
engine_(io_service, options, client_name, user_name, protocol_name, protocol_version),
namenode_(& engine_), options_(options) {}
- static Status CheckValidPermissionMask(short permissions);
+ static uint16_t GetDefaultPermissionMask();
+
+ static Status CheckValidPermissionMask(uint16_t permissions);
+
+ static Status CheckValidReplication(uint16_t replication);
void Connect(const std::string &cluster_name,
const std::vector<ResolvedNamenodeInfo> &servers,
std::function<void(const Status &)> &&handler);
- void GetBlockLocations(const std::string & path,
+ void GetBlockLocations(const std::string & path, uint64_t offset, uint64_t length,
std::function<void(const Status &, std::shared_ptr<const struct FileInfo>)> handler);
+ void GetPreferredBlockSize(const std::string & path,
+ std::function<void(const Status &, const uint64_t)> handler);
+
+ void SetReplication(const std::string & path, int16_t replication,
+ std::function<void(const Status &)> handler);
+
+ void SetTimes(const std::string & path, uint64_t mtime, uint64_t atime,
+ std::function<void(const Status &)> handler);
+
void GetFileInfo(const std::string & path,
std::function<void(const Status &, const StatInfo &)> handler);
@@ -67,7 +80,7 @@ public:
std::function<void(const Status &, std::shared_ptr<std::vector<StatInfo>>&, bool)> handler,
const std::string & start_after = "");
- void Mkdirs(const std::string & path, long permissions, bool createparent,
+ void Mkdirs(const std::string & path, uint16_t permissions, bool createparent,
std::function<void(const Status &)> handler);
void Delete(const std::string & path, bool recursive,
@@ -76,7 +89,7 @@ public:
void Rename(const std::string & oldPath, const std::string & newPath,
std::function<void(const Status &)> handler);
- void SetPermission(const std::string & path, short permissions,
+ void SetPermission(const std::string & path, uint16_t permissions,
std::function<void(const Status &)> handler);
void SetOwner(const std::string & path, const std::string & username,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a8edd4e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc
index e28e4e9..c160f7f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc
@@ -39,8 +39,8 @@ TEST_F(HdfsExtTest, TestGetBlockLocations) {
EXPECT_EQ(0, result);
// Test non-extant files
- result = hdfsGetBlockLocations(connection, "non_extant_file", &blocks);
- EXPECT_NE(0, result); // Should be an error
+ EXPECT_EQ(-1, hdfsGetBlockLocations(connection, "non_extant_file", &blocks)); // Should be an error
+ EXPECT_EQ((int) std::errc::no_such_file_or_directory, errno);
// Test an extant file
std::string filename = connection.newFile(1024);
@@ -296,7 +296,6 @@ TEST_F(HdfsExtTest, TestEOF) {
HdfsHandle connection = cluster.connect_c();
hdfsFS fs = connection.handle();
EXPECT_NE(nullptr, fs);
-
//Write to a file
errno = 0;
int size = 256;
@@ -308,28 +307,284 @@ TEST_F(HdfsExtTest, TestEOF) {
EXPECT_EQ(size, hdfsWrite(fs, file, buf, size));
free(buf);
EXPECT_EQ(0, hdfsCloseFile(fs, file));
- EXPECT_EQ(0, errno);
+ //libhdfs file operations work, but sometimes sets errno ENOENT : 2
//Test normal reading (no EOF)
char buffer[300];
- EXPECT_EQ(0, errno);
file = hdfsOpenFile(fs, path.c_str(), O_RDONLY, 0, 0, 0);
EXPECT_EQ(size, hdfsPread(fs, file, 0, buffer, sizeof(buffer)));
//Read executes correctly, but causes a warning (captured in HDFS-10595)
//and sets errno to EINPROGRESS 115 : Operation now in progress
- errno = 0;
//Test reading at offset past the EOF
EXPECT_EQ(-1, hdfsPread(fs, file, sizeof(buffer), buffer, sizeof(buffer)));
EXPECT_EQ(Status::kInvalidOffset, errno);
EXPECT_EQ(0, hdfsCloseFile(fs, file));
+}
+
+ //Testing hdfsExists
+ TEST_F(HdfsExtTest, TestExists) {
+
+ HdfsHandle connection = cluster.connect_c();
+ hdfsFS fs = connection.handle();
+ EXPECT_NE(nullptr, fs);
+ //Path not found
+ EXPECT_EQ(-1, hdfsExists(fs, "/wrong/dir/"));
+ EXPECT_EQ((int ) std::errc::no_such_file_or_directory, errno);
+
+ //Correct operation
+ std::string pathDir = "/testExistsDir";
+ EXPECT_EQ(0, hdfsCreateDirectory(fs, pathDir.c_str()));
+ EXPECT_EQ(0, hdfsExists(fs, pathDir.c_str()));
+ std::string pathFile = connection.newFile(pathDir.c_str(), 1024);
+ EXPECT_EQ(0, hdfsExists(fs, pathFile.c_str()));
+
+ //Permission denied
+ EXPECT_EQ(0, hdfsChmod(fs, pathDir.c_str(), 0700));
+ HdfsHandle connection2 = cluster.connect_c("OtherGuy");
+ hdfsFS fs2 = connection2.handle();
+ EXPECT_EQ(-1, hdfsExists(fs2, pathFile.c_str()));
+ EXPECT_EQ((int ) std::errc::permission_denied, errno);
+}
+
+//Testing Replication and Time modifications
+TEST_F(HdfsExtTest, TestReplAndTime) {
+ HdfsHandle connection = cluster.connect_c();
+ hdfsFS fs = connection.handle();
+ EXPECT_NE(nullptr, fs);
+
+ std::string path = "/wrong/dir/";
+
+ //Path not found
+ EXPECT_EQ(-1, hdfsSetReplication(fs, path.c_str(), 3));
+ EXPECT_EQ((int ) std::errc::no_such_file_or_directory, errno);
+ EXPECT_EQ(-1, hdfsUtime(fs, path.c_str(), 1000000, 1000000));
+ EXPECT_EQ((int ) std::errc::no_such_file_or_directory, errno);
+
+ //Correct operation
+ path = connection.newFile(1024);
+ EXPECT_EQ(0, hdfsSetReplication(fs, path.c_str(), 7));
+ EXPECT_EQ(0, hdfsUtime(fs, path.c_str(), 123456789, 987654321));
+ hdfsFileInfo *file_info;
+ EXPECT_NE(nullptr, file_info = hdfsGetPathInfo(fs, path.c_str()));
+ EXPECT_EQ(7, file_info->mReplication);
+ EXPECT_EQ(123456789, file_info->mLastMod);
+ EXPECT_EQ(987654321, file_info->mLastAccess);
+ hdfsFreeFileInfo(file_info, 1);
+
+ //Wrong arguments
+ EXPECT_EQ(-1, hdfsSetReplication(fs, path.c_str(), 0));
+ EXPECT_EQ((int ) std::errc::invalid_argument, errno);
+ EXPECT_EQ(-1, hdfsSetReplication(fs, path.c_str(), 513));
+ EXPECT_EQ((int ) std::errc::invalid_argument, errno);
+
+ //Permission denied
+ EXPECT_EQ(0, hdfsChmod(fs, path.c_str(), 0700));
+ HdfsHandle connection2 = cluster.connect_c("OtherGuy");
+ hdfsFS fs2 = connection2.handle();
+ EXPECT_EQ(-1, hdfsSetReplication(fs2, path.c_str(), 3));
+ EXPECT_EQ((int ) std::errc::permission_denied, errno);
+ EXPECT_EQ(-1, hdfsUtime(fs2, path.c_str(), 111111111, 222222222));
+ EXPECT_EQ((int ) std::errc::permission_denied, errno);
+}
+
+//Testing getting default block size at path
+TEST_F(HdfsExtTest, TestDefaultBlockSize) {
+ HdfsHandle connection = cluster.connect_c();
+ hdfsFS fs = connection.handle();
+ EXPECT_NE(nullptr, fs);
+
+ //Correct operation (existing path)
+ std::string path = connection.newFile(1024);
+ long block_size = hdfsGetDefaultBlockSizeAtPath(fs, path.c_str());
+ EXPECT_GT(block_size, 0);
+ hdfsFileInfo *file_info;
+ EXPECT_NE(nullptr, file_info = hdfsGetPathInfo(fs, path.c_str()));
+ EXPECT_EQ(block_size, file_info->mBlockSize);
+ hdfsFreeFileInfo(file_info, 1);
+
+ //Non-existing path
+ path = "/wrong/dir/";
+ EXPECT_GT(hdfsGetDefaultBlockSizeAtPath(fs, path.c_str()), 0);
+
+ //No path specified
+ EXPECT_GT(hdfsGetDefaultBlockSize(fs), 0);
+}
+
+//Testing getting hosts
+TEST_F(HdfsExtTest, TestHosts) {
+ HdfsHandle connection = cluster.connect_c();
+ hdfsFS fs = connection.handle();
+ EXPECT_NE(nullptr, fs);
+
+ char *** hosts = nullptr;
+
+ // Free a null pointer
+ hdfsFreeHosts(hosts);
+ EXPECT_EQ(0, errno);
+
+ // Test non-existent files
+ EXPECT_EQ(nullptr, hdfsGetHosts(fs, "/wrong/file/", 0, std::numeric_limits<int64_t>::max()));
+ EXPECT_EQ((int ) std::errc::no_such_file_or_directory, errno);
+
+ // Test an existent file
+ std::string filename = connection.newFile(1024);
+ EXPECT_NE(nullptr, hosts = hdfsGetHosts(fs, filename.c_str(), 0, std::numeric_limits<int64_t>::max()));
+
+ //Make sure there is at least one host
+ EXPECT_NE(nullptr, *hosts);
+ EXPECT_NE(nullptr, **hosts);
+
+ hdfsFreeHosts(hosts);
+ EXPECT_EQ(0, errno);
+
+ //Test invalid arguments
+ EXPECT_EQ(nullptr, hdfsGetHosts(fs, filename.c_str(), 0, std::numeric_limits<int64_t>::max()+1));
+ EXPECT_EQ((int) std::errc::invalid_argument, errno);
+
+ //Test invalid arguments
+ EXPECT_EQ(nullptr, hdfsGetHosts(fs, filename.c_str(), std::numeric_limits<int64_t>::max()+1, std::numeric_limits<int64_t>::max()));
+ EXPECT_EQ((int) std::errc::invalid_argument, errno);
+}
+
+//Testing read statistics
+TEST_F(HdfsExtTest, TestReadStats) {
+ HdfsHandle connection = cluster.connect_c();
+ hdfsFS fs = connection.handle();
+ EXPECT_NE(nullptr, fs);
+
+ struct hdfsReadStatistics *stats;
+
+ //Write to a file
+ int size = 256;
+ std::string path = "/readStatTest";
+ hdfsFile file = hdfsOpenFile(fs, path.c_str(), O_WRONLY, 0, 0, 0);
+ EXPECT_NE(nullptr, file);
+ void * buf = malloc(size);
+ bzero(buf, size);
+ EXPECT_EQ(size, hdfsWrite(fs, file, buf, size));
+ free(buf);
+ EXPECT_EQ(0, hdfsCloseFile(fs, file));
+
+ //test before reading
+ file = hdfsOpenFile(fs, path.c_str(), O_RDONLY, 0, 0, 0);
+ EXPECT_EQ(0, hdfsFileGetReadStatistics(file, &stats));
+ EXPECT_EQ(0, stats->totalBytesRead);
+ hdfsFileFreeReadStatistics(stats);
+
+ //test after reading
+ char buffer[123];
+ //Read executes correctly, but causes a warning (captured in HDFS-10595)
+ EXPECT_EQ(sizeof(buffer), hdfsRead(fs, file, buffer, sizeof(buffer)));
+ EXPECT_EQ(0, hdfsFileGetReadStatistics(file, &stats));
+ EXPECT_EQ(sizeof(buffer), stats->totalBytesRead);
+ EXPECT_EQ(sizeof(buffer), stats->totalLocalBytesRead);
+ EXPECT_EQ(0, hdfsReadStatisticsGetRemoteBytesRead(stats));
+ hdfsFileFreeReadStatistics(stats);
+
+ //test after clearing
+ EXPECT_EQ(0, hdfsFileClearReadStatistics(file));
+ EXPECT_EQ(0, hdfsFileGetReadStatistics(file, &stats));
+ EXPECT_EQ(0, stats->totalBytesRead);
+ hdfsFileFreeReadStatistics(stats);
+
+ EXPECT_EQ(0, hdfsCloseFile(fs, file));
EXPECT_EQ(0, errno);
}
+//Testing working directory
+TEST_F(HdfsExtTest, TestWorkingDirectory) {
+ HdfsHandle connection = cluster.connect_c();
+ hdfsFS fs = connection.handle();
+ EXPECT_NE(nullptr, fs);
+
+ //Correct operation of setter and getter
+ std::string pathDir = "/testWorkDir/";
+ EXPECT_EQ(0, hdfsCreateDirectory(fs, pathDir.c_str()));
+ std::string pathFile = connection.newFile(pathDir.c_str(), 1024);
+ EXPECT_EQ(0, hdfsSetWorkingDirectory(fs, pathDir.c_str()));
+ char array[100];
+ EXPECT_STREQ(pathDir.c_str(), hdfsGetWorkingDirectory(fs, array, 100));
+
+ //Get relative path
+ std::size_t slashPos = pathFile.find_last_of("/");
+ std::string fileName = pathFile.substr(slashPos + 1);
+
+ //Testing various functions with relative path:
+
+ //hdfsGetDefaultBlockSizeAtPath
+ EXPECT_GT(hdfsGetDefaultBlockSizeAtPath(fs, fileName.c_str()), 0);
+
+ //hdfsSetReplication
+ EXPECT_EQ(0, hdfsSetReplication(fs, fileName.c_str(), 7));
+
+ //hdfsUtime
+ EXPECT_EQ(0, hdfsUtime(fs, fileName.c_str(), 123456789, 987654321));
+
+ //hdfsExists
+ EXPECT_EQ(0, hdfsExists(fs, fileName.c_str()));
+
+ //hdfsGetPathInfo
+ hdfsFileInfo *file_info;
+ EXPECT_NE(nullptr, file_info = hdfsGetPathInfo(fs, fileName.c_str()));
+ hdfsFreeFileInfo(file_info, 1);
+
+ //hdfsOpenFile
+ hdfsFile file;
+ file = hdfsOpenFile(fs, fileName.c_str(), O_RDONLY, 0, 0, 0);
+ EXPECT_EQ(0, hdfsCloseFile(fs, file));
+
+ //hdfsCreateDirectory
+ EXPECT_EQ(0, hdfsCreateDirectory(fs, "newDir"));
+
+ //add another file
+ std::string fileName2 = connection.newFile(pathDir + "/newDir", 1024);
+
+ //hdfsListDirectory
+ int numEntries;
+ hdfsFileInfo * dirList;
+ EXPECT_NE(nullptr, dirList = hdfsListDirectory(fs, "newDir", &numEntries));
+ EXPECT_EQ(1, numEntries);
+ hdfsFreeFileInfo(dirList, 1);
+
+ //hdfsChmod
+ EXPECT_EQ(0, hdfsChmod(fs, fileName.c_str(), 0777));
+
+ //hdfsChown
+ EXPECT_EQ(0, hdfsChown(fs, fileName.c_str(), "cool", "nice"));
+
+ //hdfsDisallowSnapshot
+ EXPECT_EQ(0, hdfsDisallowSnapshot(fs, "newDir"));
+
+ //hdfsAllowSnapshot
+ EXPECT_EQ(0, hdfsAllowSnapshot(fs, "newDir"));
+
+ //hdfsCreateSnapshot
+ EXPECT_EQ(0, hdfsCreateSnapshot(fs, "newDir", "Some"));
+
+ //hdfsDeleteSnapshot
+ EXPECT_EQ(0, hdfsDeleteSnapshot(fs, "newDir", "Some"));
+
+ //hdfsGetBlockLocations
+ hdfsBlockLocations * blocks = nullptr;
+ EXPECT_EQ(0, hdfsGetBlockLocations(connection, fileName.c_str(), &blocks));
+ hdfsFreeBlockLocations(blocks);
+
+ //hdfsGetHosts
+ char *** hosts;
+ EXPECT_NE(nullptr, hosts = hdfsGetHosts(fs, fileName.c_str(), 0, std::numeric_limits<int64_t>::max()));
+ hdfsFreeHosts(hosts);
+
+ //hdfsRename
+ EXPECT_EQ(0, hdfsRename(fs, fileName.c_str(), "new_file_name"));
+
+ //hdfsDelete
+ EXPECT_EQ(0, hdfsDelete(fs, "new_file_name", 0));
}
+}
int main(int argc, char *argv[]) {
// The following line must be executed to initialize Google Mock
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a8edd4e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c
index 32f920f..fd82da3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c
@@ -50,25 +50,25 @@ int hdfsFileIsOpenForWrite(hdfsFile file) {
return libhdfs_hdfsFileIsOpenForWrite(file->libhdfsRep);
}
-int hdfsFileGetReadStatistics(hdfsFile file,
- struct hdfsReadStatistics **stats) {
- return libhdfs_hdfsFileGetReadStatistics
- (file->libhdfsRep, (struct libhdfs_hdfsReadStatistics **)stats);
+int hdfsFileGetReadStatistics(hdfsFile file, struct hdfsReadStatistics **stats) {
+ //We do not track which bytes were remote or local, so we assume all are local
+ int ret = libhdfspp_hdfsFileGetReadStatistics(file->libhdfsppRep, (struct libhdfspp_hdfsReadStatistics **)stats);
+ if(!ret) {
+ (*stats)->totalLocalBytesRead = (*stats)->totalBytesRead;
+ }
+ return ret;
}
-int64_t hdfsReadStatisticsGetRemoteBytesRead(
- const struct hdfsReadStatistics *stats) {
- return libhdfs_hdfsReadStatisticsGetRemoteBytesRead
- ((struct libhdfs_hdfsReadStatistics *)stats);
+int64_t hdfsReadStatisticsGetRemoteBytesRead(const struct hdfsReadStatistics *stats) {
+ return libhdfspp_hdfsReadStatisticsGetRemoteBytesRead((struct libhdfspp_hdfsReadStatistics *)stats);
}
int hdfsFileClearReadStatistics(hdfsFile file) {
- return libhdfs_hdfsFileClearReadStatistics(file->libhdfsRep);
+ return libhdfspp_hdfsFileClearReadStatistics(file->libhdfsppRep);
}
void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats) {
- libhdfs_hdfsFileFreeReadStatistics(
- (struct libhdfs_hdfsReadStatistics *)stats);
+ libhdfspp_hdfsFileFreeReadStatistics((struct libhdfspp_hdfsReadStatistics *)stats);
}
hdfsFS hdfsConnectAsUser(const char* nn, tPort port, const char *user) {
@@ -208,15 +208,15 @@ int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key,
}
int hdfsConfGetStr(const char *key, char **val) {
- return libhdfs_hdfsConfGetStr(key, val);
+ return libhdfspp_hdfsConfGetStr(key, val);
}
int hdfsConfGetInt(const char *key, int32_t *val) {
- return libhdfs_hdfsConfGetInt(key, val);
+ return libhdfspp_hdfsConfGetInt(key, val);
}
void hdfsConfStrFree(char *val) {
- libhdfs_hdfsConfStrFree(val);
+ libhdfspp_hdfsConfStrFree(val);
}
int hdfsDisconnect(hdfsFS fs) {
@@ -269,15 +269,30 @@ int hdfsCloseFile(hdfsFS fs, hdfsFile file) {
}
int hdfsExists(hdfsFS fs, const char *path) {
- return libhdfs_hdfsExists(fs->libhdfsRep, path);
+ return libhdfspp_hdfsExists(fs->libhdfsppRep, path);
}
int hdfsSeek(hdfsFS fs, hdfsFile file, tOffset desiredPos) {
- return libhdfs_hdfsSeek(fs->libhdfsRep, file->libhdfsRep, desiredPos);
+ int ret1 = libhdfs_hdfsSeek(fs->libhdfsRep, file->libhdfsRep, desiredPos);
+ int ret2 = libhdfspp_hdfsSeek(fs->libhdfsppRep, file->libhdfsppRep, desiredPos);
+ if (ret1) {
+ return ret1;
+ } else if (ret2) {
+ return ret2;
+ } else {
+ return 0;
+ }
}
tOffset hdfsTell(hdfsFS fs, hdfsFile file) {
- return libhdfs_hdfsTell(fs->libhdfsRep, file->libhdfsRep);
+ tOffset ret1 = libhdfs_hdfsTell(fs->libhdfsRep, file->libhdfsRep);
+ tOffset ret2 = libhdfspp_hdfsTell(fs->libhdfsppRep, file->libhdfsppRep);
+ if (ret1 != ret2) {
+ errno = EIO;
+ return -1;
+ } else {
+ return ret1;
+ }
}
tSize hdfsRead(hdfsFS fs, hdfsFile file, void* buffer, tSize length) {
@@ -320,7 +335,7 @@ int hdfsHSync(hdfsFS fs, hdfsFile file) {
}
int hdfsAvailable(hdfsFS fs, hdfsFile file) {
- return libhdfs_hdfsAvailable(fs->libhdfsRep, file->libhdfsRep);
+ return libhdfspp_hdfsAvailable(fs->libhdfsppRep, file->libhdfsppRep);
}
int hdfsCopy(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst) {
@@ -340,11 +355,19 @@ int hdfsRename(hdfsFS fs, const char* oldPath, const char* newPath) {
}
char* hdfsGetWorkingDirectory(hdfsFS fs, char *buffer, size_t bufferSize) {
- return libhdfs_hdfsGetWorkingDirectory(fs->libhdfsRep, buffer, bufferSize);
+ return libhdfspp_hdfsGetWorkingDirectory(fs->libhdfsppRep, buffer, bufferSize);
}
int hdfsSetWorkingDirectory(hdfsFS fs, const char* path) {
- return libhdfs_hdfsSetWorkingDirectory(fs->libhdfsRep, path);
+ int ret1 = libhdfspp_hdfsSetWorkingDirectory(fs->libhdfsppRep, path);
+ int ret2 = libhdfs_hdfsSetWorkingDirectory(fs->libhdfsRep, path);
+ if (ret1) {
+ return ret1;
+ } else if (ret2) {
+ return ret2;
+ } else {
+ return 0;
+ }
}
int hdfsCreateDirectory(hdfsFS fs, const char* path) {
@@ -352,7 +375,7 @@ int hdfsCreateDirectory(hdfsFS fs, const char* path) {
}
int hdfsSetReplication(hdfsFS fs, const char* path, int16_t replication) {
- return libhdfs_hdfsSetReplication(fs->libhdfsRep, path, replication);
+ return libhdfspp_hdfsSetReplication(fs->libhdfsppRep, path, replication);
}
hdfsFileInfo *hdfsListDirectory(hdfsFS fs, const char* path,
@@ -376,19 +399,19 @@ int hdfsFileIsEncrypted(hdfsFileInfo *hdfsFileInfo) {
char*** hdfsGetHosts(hdfsFS fs, const char* path,
tOffset start, tOffset length) {
- return libhdfs_hdfsGetHosts(fs->libhdfsRep, path, start, length);
+ return libhdfspp_hdfsGetHosts(fs->libhdfsppRep, path, start, length);
}
void hdfsFreeHosts(char ***blockHosts) {
- return libhdfs_hdfsFreeHosts(blockHosts);
+ return libhdfspp_hdfsFreeHosts(blockHosts);
}
tOffset hdfsGetDefaultBlockSize(hdfsFS fs) {
- return libhdfs_hdfsGetDefaultBlockSize(fs->libhdfsRep);
+ return libhdfspp_hdfsGetDefaultBlockSize(fs->libhdfsppRep);
}
tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path) {
- return libhdfs_hdfsGetDefaultBlockSizeAtPath(fs->libhdfsRep, path);
+ return libhdfspp_hdfsGetDefaultBlockSizeAtPath(fs->libhdfsppRep, path);
}
tOffset hdfsGetCapacity(hdfsFS fs) {
@@ -409,7 +432,7 @@ int hdfsChmod(hdfsFS fs, const char* path, short mode) {
}
int hdfsUtime(hdfsFS fs, const char* path, tTime mtime, tTime atime) {
- return libhdfs_hdfsUtime(fs->libhdfsRep, path, mtime, atime);
+ return libhdfspp_hdfsUtime(fs->libhdfsppRep, path, mtime, atime);
}
struct hadoopRzOptions *hadoopRzOptionsAlloc(void) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/52] [abbrv] hadoop git commit: HDFS-10672: libhdfs++: reorder
directories in src/main/libhdfspp/examples,
and add C++ version of cat tool. Contributed by Anatoli Shein.
Posted by jh...@apache.org.
HDFS-10672: libhdfs++: reorder directories in src/main/libhdfspp/examples, and add C++ version of cat tool. Contributed by Anatoli Shein.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/649aff11
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/649aff11
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/649aff11
Branch: refs/heads/HDFS-8707
Commit: 649aff11fe618fee642fe4e617b94a581602a258
Parents: 4cb0dad
Author: James <jh...@apache.org>
Authored: Thu Aug 4 18:09:49 2016 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../native/libhdfspp/examples/CMakeLists.txt | 3 +-
.../native/libhdfspp/examples/c/CMakeLists.txt | 19 +++
.../libhdfspp/examples/c/cat/CMakeLists.txt | 35 ++++
.../main/native/libhdfspp/examples/c/cat/cat.c | 121 ++++++++++++++
.../libhdfspp/examples/cat/CMakeLists.txt | 19 ---
.../libhdfspp/examples/cat/c/CMakeLists.txt | 27 ----
.../main/native/libhdfspp/examples/cat/c/cat.c | 162 -------------------
.../libhdfspp/examples/cpp/CMakeLists.txt | 19 +++
.../libhdfspp/examples/cpp/cat/CMakeLists.txt | 35 ++++
.../native/libhdfspp/examples/cpp/cat/cat.cpp | 123 ++++++++++++++
.../native/libhdfspp/include/hdfspp/hdfspp.h | 12 +-
.../native/libhdfspp/include/hdfspp/status.h | 4 +
.../native/libhdfspp/lib/bindings/c/hdfs.cc | 12 +-
.../main/native/libhdfspp/lib/common/status.cc | 4 +
.../main/native/libhdfspp/lib/common/util.cc | 5 +
.../main/native/libhdfspp/lib/common/util_c.h | 31 ++++
.../main/native/libhdfspp/lib/fs/filehandle.cc | 28 ++--
.../main/native/libhdfspp/lib/fs/filehandle.h | 32 ++--
.../native/libhdfspp/tests/bad_datanode_test.cc | 4 +
.../native/libhdfspp/tests/hdfs_ext_test.cc | 37 ++++-
20 files changed, 482 insertions(+), 250 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649aff11/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/CMakeLists.txt
index 76880cd..6b680cc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/CMakeLists.txt
@@ -16,4 +16,5 @@
# limitations under the License.
#
-add_subdirectory(cat)
+add_subdirectory(c)
+add_subdirectory(cpp)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649aff11/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/CMakeLists.txt
new file mode 100644
index 0000000..76880cd
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/CMakeLists.txt
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_subdirectory(cat)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649aff11/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/CMakeLists.txt
new file mode 100644
index 0000000..1319293
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/CMakeLists.txt
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Default LIBHDFSPP_DIR to the default install location. You can override
+# it by add -DLIBHDFSPP_DIR=... to your cmake invocation
+set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX})
+
+include_directories( ${LIBHDFSPP_DIR}/include )
+link_directories( ${LIBHDFSPP_DIR}/lib )
+
+add_executable(cat_c cat.c)
+target_link_libraries(cat_c hdfspp uriparser2)
+
+# Several examples in different languages need to produce executables with
+# same names. To allow executables with same names we keep their CMake
+# names different, but specify their executable names as follows:
+set_target_properties( cat_c
+ PROPERTIES
+ OUTPUT_NAME "cat"
+)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649aff11/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/cat.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/cat.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/cat.c
new file mode 100644
index 0000000..586e0a9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/cat.c
@@ -0,0 +1,121 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+/*
+ A a stripped down version of unix's "cat".
+ Doesn't deal with any flags for now, will just attempt to read the whole file.
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "hdfspp/hdfs_ext.h"
+#include "uriparser2/uriparser2.h"
+#include "common/util_c.h"
+
+#define SCHEME "hdfs"
+
+int main(int argc, char** argv) {
+
+ char error_text[1024];
+ if (argc != 2) {
+ fprintf(stderr, "usage: cat [hdfs://[<hostname>:<port>]]/<path-to-file>\n");
+ return 1;
+ }
+
+ URI * uri = NULL;
+ const char * uri_path = argv[1];
+
+ //Separate check for scheme is required, otherwise uriparser2.h library causes memory issues under valgrind
+ const char * scheme_end = strstr(uri_path, "://");
+ if (scheme_end) {
+ if (strncmp(uri_path, SCHEME, strlen(SCHEME)) != 0) {
+ fprintf(stderr, "Scheme %.*s:// is not supported.\n", (int) (scheme_end - uri_path), uri_path);
+ return 1;
+ } else {
+ uri = uri_parse(uri_path);
+ }
+ }
+ if (!uri) {
+ fprintf(stderr, "Malformed URI: %s\n", uri_path);
+ return 1;
+ }
+
+ struct hdfsBuilder* builder = hdfsNewBuilder();
+ if (uri->host)
+ hdfsBuilderSetNameNode(builder, uri->host);
+ if (uri->port != 0)
+ hdfsBuilderSetNameNodePort(builder, uri->port);
+
+ hdfsFS fs = hdfsBuilderConnect(builder);
+ if (fs == NULL) {
+ hdfsGetLastError(error_text, sizeof(error_text));
+ const char * host = uri->host ? uri->host : "<default>";
+ int port = uri->port;
+ if (port == 0)
+ port = 8020;
+ fprintf(stderr, "Unable to connect to %s:%d, hdfsConnect returned null.\n%s\n",
+ host, port, error_text);
+ return 1;
+ }
+
+ hdfsFile file = hdfsOpenFile(fs, uri->path, 0, 0, 0, 0);
+ if (NULL == file) {
+ hdfsGetLastError(error_text, sizeof(error_text));
+ fprintf(stderr, "Unable to open file %s: %s\n", uri->path, error_text );
+ hdfsDisconnect(fs);
+ hdfsFreeBuilder(builder);
+ return 1;
+ }
+
+ char input_buffer[4096];
+
+ ssize_t read_bytes_count = 0;
+ ssize_t last_read_bytes = 0;
+
+ while (0 < (last_read_bytes =
+ hdfsPread(fs, file, read_bytes_count, input_buffer, sizeof(input_buffer)))) {
+ fwrite(input_buffer, last_read_bytes, 1, stdout);
+ read_bytes_count += last_read_bytes;
+ }
+
+ int res = 0;
+ res = hdfsCloseFile(fs, file);
+ if (0 != res) {
+ hdfsGetLastError(error_text, sizeof(error_text));
+ fprintf(stderr, "Error closing file: %s\n", error_text);
+ hdfsDisconnect(fs);
+ hdfsFreeBuilder(builder);
+ return 1;
+ }
+
+ res = hdfsDisconnect(fs);
+ if (0 != res) {
+ hdfsGetLastError(error_text, sizeof(error_text));
+ fprintf(stderr, "Error disconnecting filesystem: %s", error_text);
+ hdfsFreeBuilder(builder);
+ return 1;
+ }
+
+ hdfsFreeBuilder(builder);
+ free(uri);
+ // Clean up static data and prevent valgrind memory leaks
+ ShutdownProtobufLibrary_C();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649aff11/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cat/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cat/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cat/CMakeLists.txt
deleted file mode 100644
index 93139ce..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cat/CMakeLists.txt
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-add_subdirectory(c)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649aff11/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cat/c/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cat/c/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cat/c/CMakeLists.txt
deleted file mode 100644
index a2dc4a4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cat/c/CMakeLists.txt
+++ /dev/null
@@ -1,27 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Default LIBHDFSPP_DIR to the default install location. You can override
-# it by add -DLIBHDFSPP_DIR=... to your cmake invocation
-set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX})
-
-include_directories( ${LIBHDFSPP_DIR}/include )
-link_directories( ${LIBHDFSPP_DIR}/lib )
-
-add_executable(cat cat.c)
-target_link_libraries(cat hdfspp)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649aff11/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cat/c/cat.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cat/c/cat.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cat/c/cat.c
deleted file mode 100644
index dec8758..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cat/c/cat.c
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-*/
-
-/*
- A a stripped down version of unix's "cat".
- Doesn't deal with any flags for now, will just attempt to read the whole file.
-*/
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-
-#include "hdfspp/hdfs_ext.h"
-
-#define SCHEME "hdfs"
-#define MAX_STRING 1024
-
-struct Uri {
- int valid;
- char host[MAX_STRING];
- int port;
- char path[MAX_STRING];
-};
-
-int min(int a, int b) {
- return a < b ? a : b;
-}
-
-void parse_uri(const char * uri_string, struct Uri * uri) {
- uri->valid = 0;
- uri->host[0] = 0;
- uri->port = -1;
- uri->path[0] = 0;
-
- // most start with hdfs scheme
- const char * remaining;
- const char * scheme_end = strstr(uri_string, "://");
- if (scheme_end != NULL) {
- if (strncmp(uri_string, SCHEME, strlen(SCHEME)) != 0)
- return;
-
- remaining = scheme_end + 3;
-
- // parse authority
- const char * authority_end = strstr(remaining, "/");
- if (authority_end != NULL) {
- char authority[MAX_STRING];
- strncpy(authority, remaining, min(authority_end - remaining, sizeof(authority)));
- remaining = authority_end;
-
- char * host_port_separator = strstr(authority, ":");
- if (host_port_separator != NULL) {
- errno = 0;
- uri->port = strtol(host_port_separator + 1, NULL, 10);
- if (errno != 0)
- return;
-
- // Terminate authority at the new end of the host
- *host_port_separator = 0;
- }
- strncpy(uri->host, authority, sizeof(uri->host));
- }
- strncpy(uri->path, remaining, sizeof(uri->path));
- } else {
- // Absolute path
- strncpy(uri->path, uri_string, sizeof(uri->path));
- }
-
- uri->valid = 1;
-};
-
-int main(int argc, char** argv) {
-
- char error_text[1024];
- if (argc != 2) {
- fprintf(stderr, "usage: cat [hdfs://[<hostname>:<port>]]/<path-to-file>\n");
- return 1;
- }
-
- const char * uri_path = argv[1];
- struct Uri uri;
- parse_uri(uri_path, &uri);
- if (!uri.valid) {
- fprintf(stderr, "malformed URI: %s\n", uri_path);
- return 1;
- }
-
- struct hdfsBuilder* builder = hdfsNewBuilder();
- if (*uri.host != 0)
- hdfsBuilderSetNameNode(builder, uri.host);
- if (uri.port != -1)
- hdfsBuilderSetNameNodePort(builder, uri.port);
-
- hdfsFS fs = hdfsBuilderConnect(builder);
- if (fs == NULL) {
- hdfsGetLastError(error_text, sizeof(error_text));
- const char * host = uri.host[0] ? uri.host : "<default>";
- int port = uri.port;
- if (-1 == port)
- port = 8020;
- fprintf(stderr, "Unable to connect to %s:%d, hdfsConnect returned null.\n%s\n",
- host, port, error_text);
- return 1;
- }
-
- hdfsFile file = hdfsOpenFile(fs, uri.path, 0, 0, 0, 0);
- if (NULL == file) {
- hdfsGetLastError(error_text, sizeof(error_text));
- fprintf(stderr, "Unable to open file %s: %s\n", uri.path, error_text );
- hdfsDisconnect(fs);
- hdfsFreeBuilder(builder);
- return 1;
- }
-
- char input_buffer[4096];
-
- ssize_t read_bytes_count = 0;
- ssize_t last_read_bytes = 0;
-
- while (0 < (last_read_bytes =
- hdfsPread(fs, file, read_bytes_count, input_buffer, sizeof(input_buffer)))) {
- fwrite(input_buffer, last_read_bytes, 1, stdout);
- read_bytes_count += last_read_bytes;
- }
-
- int res = 0;
- res = hdfsCloseFile(fs, file);
- if (0 != res) {
- hdfsGetLastError(error_text, sizeof(error_text));
- fprintf(stderr, "Error closing file: %s\n", error_text);
- hdfsDisconnect(fs);
- hdfsFreeBuilder(builder);
- return 1;
- }
-
- res = hdfsDisconnect(fs);
- if (0 != res) {
- hdfsGetLastError(error_text, sizeof(error_text));
- fprintf(stderr, "Error disconnecting filesystem: %s", error_text);
- hdfsFreeBuilder(builder);
- return 1;
- }
-
- hdfsFreeBuilder(builder);
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649aff11/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/CMakeLists.txt
new file mode 100644
index 0000000..76880cd
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/CMakeLists.txt
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_subdirectory(cat)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649aff11/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/cat/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/cat/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/cat/CMakeLists.txt
new file mode 100644
index 0000000..9ec3332
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/cat/CMakeLists.txt
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Default LIBHDFSPP_DIR to the default install location. You can override
+# it by add -DLIBHDFSPP_DIR=... to your cmake invocation
+set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX})
+
+include_directories( ${LIBHDFSPP_DIR}/include )
+link_directories( ${LIBHDFSPP_DIR}/lib )
+
+add_executable(cat_cpp cat.cpp)
+target_link_libraries(cat_cpp hdfspp)
+
+# Several examples in different languages need to produce executables with
+# same names. To allow executables with same names we keep their CMake
+# names different, but specify their executable names as follows:
+set_target_properties( cat_cpp
+ PROPERTIES
+ OUTPUT_NAME "cat"
+)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649aff11/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/cat/cat.cpp
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/cat/cat.cpp b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/cat/cat.cpp
new file mode 100644
index 0000000..bfab507
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/cat/cat.cpp
@@ -0,0 +1,123 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+/*
+ A a stripped down version of unix's "cat".
+ Doesn't deal with any flags for now, will just attempt to read the whole file.
+*/
+
+#include "hdfspp/hdfspp.h"
+#include "common/hdfs_configuration.h"
+#include "common/configuration_loader.h"
+#include "common/uri.h"
+
+#include <google/protobuf/io/coded_stream.h>
+
+using namespace std;
+using namespace hdfs;
+
+#define SCHEME "hdfs"
+
+int main(int argc, char *argv[]) {
+ if (argc != 2) {
+ cerr << "usage: cat [hdfs://[<hostname>:<port>]]/<path-to-file>" << endl;
+ return 1;
+ }
+
+ optional<URI> uri;
+ const string uri_path = argv[1];
+
+ //Separate check for scheme is required, otherwise common/uri.h library causes memory issues under valgrind
+ size_t scheme_end = uri_path.find("://");
+ if (scheme_end != string::npos) {
+ if(uri_path.substr(0, string(SCHEME).size()).compare(SCHEME) != 0) {
+ cerr << "Scheme " << uri_path.substr(0, scheme_end) << ":// is not supported" << endl;
+ return 1;
+ } else {
+ uri = URI::parse_from_string(uri_path);
+ }
+ }
+ if (!uri) {
+ cerr << "Malformed URI: " << uri_path << endl;
+ return 1;
+ }
+
+ ConfigurationLoader loader;
+ optional<HdfsConfiguration> config = loader.LoadDefaultResources<HdfsConfiguration>();
+ const char * envHadoopConfDir = getenv("HADOOP_CONF_DIR");
+ if (envHadoopConfDir && (*envHadoopConfDir != 0) ) {
+ config = loader.OverlayResourceFile(*config, string(envHadoopConfDir) + "/core-site.xml");
+ }
+
+ Options options;
+ if(config){
+ options = config->GetOptions();
+ }
+
+ IoService * io_service = IoService::New();
+
+ FileSystem *fs_raw = FileSystem::New(io_service, "", options);
+ if (!fs_raw) {
+ cerr << "Could not create FileSystem object" << endl;
+ return 1;
+ }
+ //wrapping fs_raw into a unique pointer to guarantee deletion
+ unique_ptr<FileSystem> fs(fs_raw);
+
+ Status stat = fs->Connect(uri->get_host(), to_string(*(uri->get_port())));
+ if (!stat.ok()) {
+ cerr << "Could not connect to " << uri->get_host() << ":" << *(uri->get_port()) << endl;
+ return 1;
+ }
+
+ FileHandle *file_raw = nullptr;
+ stat = fs->Open(uri->get_path(), &file_raw);
+ if (!stat.ok()) {
+ cerr << "Could not open file " << uri->get_path() << endl;
+ return 1;
+ }
+ //wrapping file_raw into a unique pointer to guarantee deletion
+ unique_ptr<FileHandle> file(file_raw);
+
+ char input_buffer[4096];
+ ssize_t read_bytes_count = 0;
+ size_t last_read_bytes = 0;
+
+ do{
+ //Reading file chunks
+ Status stat = file->PositionRead(input_buffer, sizeof(input_buffer), read_bytes_count, &last_read_bytes);
+ if(stat.ok()) {
+ //Writing file chunks to stdout
+ fwrite(input_buffer, last_read_bytes, 1, stdout);
+ read_bytes_count += last_read_bytes;
+ } else {
+ if(stat.is_invalid_offset()){
+ //Reached the end of the file
+ break;
+ } else {
+ cerr << "Error reading the file: " << stat.ToString() << endl;
+ return 1;
+ }
+ }
+ } while (last_read_bytes > 0);
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649aff11/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
index a44df74..20a651a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
@@ -82,18 +82,18 @@ public:
* stops at the block boundary.
*
* @param buf the pointer to the buffer
- * @param nbyte the size of the buffer
+ * @param buf_size the size of the buffer
* @param offset the offset the file
*
* The handler returns the datanode that serves the block and the number of
- * bytes has read.
+ * bytes has read. Status::InvalidOffset is returned when trying to begin
+ * a read past the EOF.
**/
virtual void
- PositionRead(void *buf, size_t nbyte, uint64_t offset,
+ PositionRead(void *buf, size_t buf_size, uint64_t offset,
const std::function<void(const Status &, size_t)> &handler) = 0;
-
- virtual Status PositionRead(void *buf, size_t *nbyte, off_t offset) = 0;
- virtual Status Read(void *buf, size_t *nbyte) = 0;
+ virtual Status PositionRead(void *buf, size_t buf_size, off_t offset, size_t *bytes_read) = 0;
+ virtual Status Read(void *buf, size_t buf_size, size_t *bytes_read) = 0;
virtual Status Seek(off_t *offset, std::ios_base::seekdir whence) = 0;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649aff11/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
index f217cad..0187786 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
@@ -45,10 +45,13 @@ class Status {
static Status AuthenticationFailed();
static Status Canceled();
static Status PathNotFound(const char *msg);
+ static Status InvalidOffset(const char *msg);
// success
bool ok() const { return code_ == 0; }
+ bool is_invalid_offset() const { return code_ == kInvalidOffset; }
+
// Returns the string "OK" for success.
std::string ToString() const;
@@ -73,6 +76,7 @@ class Status {
kAccessControlException = 258,
kStandbyException = 259,
kSnapshotProtocolException = 260,
+ kInvalidOffset = 261,
};
std::string get_exception_class_str() const {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649aff11/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
index a42feae..be57a7e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
@@ -173,6 +173,10 @@ static int Error(const Status &stat) {
errnum = ENOTEMPTY;
default_message = "Directory is not empty";
break;
+ case Status::Code::kInvalidOffset:
+ errnum = Status::Code::kInvalidOffset;
+ default_message = "Trying to begin a read past the EOF";
+ break;
default:
errnum = ENOSYS;
default_message = "Error: unrecognised code";
@@ -754,8 +758,8 @@ tSize hdfsPread(hdfsFS fs, hdfsFile file, tOffset position, void *buffer,
return -1;
}
- size_t len = length;
- Status stat = file->get_impl()->PositionRead(buffer, &len, position);
+ size_t len = 0;
+ Status stat = file->get_impl()->PositionRead(buffer, length, position, &len);
if(!stat.ok()) {
return Error(stat);
}
@@ -775,8 +779,8 @@ tSize hdfsRead(hdfsFS fs, hdfsFile file, void *buffer, tSize length) {
return -1;
}
- size_t len = length;
- Status stat = file->get_impl()->Read(buffer, &len);
+ size_t len = 0;
+ Status stat = file->get_impl()->Read(buffer, length, &len);
if (!stat.ok()) {
return Error(stat);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649aff11/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
index 796b1a2..b351900 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
@@ -127,6 +127,10 @@ Status Status::Canceled() {
return Status(kOperationCanceled, "Operation canceled");
}
+Status Status::InvalidOffset(const char *msg){
+ return Status(kInvalidOffset, msg);
+}
+
std::string Status::ToString() const {
if (code_ == kOk) {
return "OK";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649aff11/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
index 6fd5a7c..7bb1e30 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
@@ -17,6 +17,7 @@
*/
#include "common/util.h"
+#include "common/util_c.h"
#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
#include <exception>
@@ -107,3 +108,7 @@ std::string SafeDisconnect(asio::ip::tcp::socket *sock) {
}
}
+
+void ShutdownProtobufLibrary_C() {
+ google::protobuf::ShutdownProtobufLibrary();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649aff11/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util_c.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util_c.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util_c.h
new file mode 100644
index 0000000..c7db7d2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util_c.h
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef LIB_COMMON_UTIL_C_H_
+#define LIB_COMMON_UTIL_C_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ void ShutdownProtobufLibrary_C();
+
+#ifdef __cplusplus
+} /* end extern "C" */
+#endif
+
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649aff11/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
index 40f1b4a..f40b81c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
@@ -48,11 +48,11 @@ FileHandleImpl::FileHandleImpl(const std::string & cluster_name,
}
void FileHandleImpl::PositionRead(
- void *buf, size_t nbyte, uint64_t offset,
+ void *buf, size_t buf_size, uint64_t offset,
const std::function<void(const Status &, size_t)> &handler) {
LOG_TRACE(kFileHandle, << "FileHandleImpl::PositionRead("
<< FMT_THIS_ADDR << ", buf=" << buf
- << ", nbyte=" << nbyte << ") called");
+ << ", buf_size=" << buf_size << ") called");
/* prevent usage after cancelation */
if(cancel_state_->is_canceled()) {
@@ -71,13 +71,14 @@ void FileHandleImpl::PositionRead(
handler(status, bytes_read);
};
- AsyncPreadSome(offset, asio::buffer(buf, nbyte), bad_node_tracker_, callback);
+ AsyncPreadSome(offset, asio::buffer(buf, buf_size), bad_node_tracker_, callback);
}
-Status FileHandleImpl::PositionRead(void *buf, size_t *nbyte, off_t offset) {
+Status FileHandleImpl::PositionRead(void *buf, size_t buf_size, off_t offset, size_t *bytes_read) {
LOG_TRACE(kFileHandle, << "FileHandleImpl::[sync]PositionRead("
<< FMT_THIS_ADDR << ", buf=" << buf
- << ", nbyte=" << *nbyte << ") called");
+ << ", buf_size=" << buf_size
+ << ", offset=" << offset << ") called");
auto callstate = std::make_shared<std::promise<std::tuple<Status, size_t>>>();
std::future<std::tuple<Status, size_t>> future(callstate->get_future());
@@ -87,7 +88,7 @@ Status FileHandleImpl::PositionRead(void *buf, size_t *nbyte, off_t offset) {
callstate->set_value(std::make_tuple(s,bytes));
};
- PositionRead(buf, *nbyte, offset, callback);
+ PositionRead(buf, buf_size, offset, callback);
/* wait for async to finish */
auto returnstate = future.get();
@@ -97,21 +98,21 @@ Status FileHandleImpl::PositionRead(void *buf, size_t *nbyte, off_t offset) {
return stat;
}
- *nbyte = std::get<1>(returnstate);
+ *bytes_read = std::get<1>(returnstate);
return stat;
}
-Status FileHandleImpl::Read(void *buf, size_t *nbyte) {
+Status FileHandleImpl::Read(void *buf, size_t buf_size, size_t *bytes_read) {
LOG_TRACE(kFileHandle, << "FileHandleImpl::Read("
<< FMT_THIS_ADDR << ", buf=" << buf
- << ", nbyte=" << *nbyte << ") called");
+ << ", buf_size=" << buf_size << ") called");
- Status stat = PositionRead(buf, nbyte, offset_);
+ Status stat = PositionRead(buf, buf_size, offset_, bytes_read);
if(!stat.ok()) {
return stat;
}
- offset_ += *nbyte;
+ offset_ += *bytes_read;
return Status::OK();
}
@@ -179,6 +180,11 @@ void FileHandleImpl::AsyncPreadSome(
return;
}
+ if(offset >= file_info_->file_length_){
+ handler(Status::InvalidOffset("AsyncPreadSome: trying to begin a read past the EOF"), "", 0);
+ return;
+ }
+
/**
* Note: block and chosen_dn will end up pointing to things inside
* the blocks_ vector. They shouldn't be directly deleted.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649aff11/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
index 57cf4b7..38c1fec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
@@ -59,32 +59,27 @@ public:
std::shared_ptr<LibhdfsEvents> event_handlers);
/*
- * [Some day reliably] Reads a particular offset into the data file.
- * On error, bytes_read returns the number of bytes successfully read; on
- * success, bytes_read will equal nbyte
+ * Reads the file at the specified offset into the buffer.
+ * bytes_read returns the number of bytes successfully read on success
+ * and on error. Status::InvalidOffset is returned when trying to begin
+ * a read past the EOF.
*/
void PositionRead(
void *buf,
- size_t nbyte,
+ size_t buf_size,
uint64_t offset,
const std::function<void(const Status &status, size_t bytes_read)> &handler
) override;
/**
- * Note: The nbyte argument for Read and Pread as well as the
- * offset argument for Seek are in/out parameters.
- *
- * For Read and Pread the value referenced by nbyte should
- * be set to the number of bytes to read. Before returning
- * the value referenced will be set by the callee to the number
- * of bytes that was successfully read.
- *
- * For Seek the value referenced by offset should be the number
- * of bytes to shift from the specified whence position. The
- * referenced value will be set to the new offset before returning.
- **/
- Status PositionRead(void *buf, size_t *bytes_read, off_t offset) override;
- Status Read(void *buf, size_t *nbyte) override;
+ * Reads the file at the specified offset into the buffer.
+ * @param buf output buffer
+ * @param buf_size size of the output buffer
+ * @param offset offset at which to start reading
+ * @param bytes_read number of bytes successfully read
+ */
+ Status PositionRead(void *buf, size_t buf_size, off_t offset, size_t *bytes_read) override;
+ Status Read(void *buf, size_t buf_size, size_t *bytes_read) override;
Status Seek(off_t *offset, std::ios_base::seekdir whence) override;
@@ -95,6 +90,7 @@ public:
* If an error occurs during connection or transfer, the callback will be
* called with bytes_read equal to the number of bytes successfully transferred.
* If no data nodes can be found, status will be Status::ResourceUnavailable.
+ * If trying to begin a read past the EOF, status will be Status::InvalidOffset.
*
*/
void AsyncPreadSome(size_t offset, const MutableBuffers &buffers,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649aff11/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc
index 9e3aeb7..5185484 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc
@@ -113,6 +113,7 @@ protected:
TEST(BadDataNodeTest, TestNoNodes) {
auto file_info = std::make_shared<struct FileInfo>();
+ file_info->file_length_ = 1; //To avoid running into EOF
file_info->blocks_.push_back(LocatedBlockProto());
LocatedBlockProto & block = file_info->blocks_[0];
ExtendedBlockProto *b = block.mutable_b();
@@ -152,6 +153,7 @@ TEST(BadDataNodeTest, TestNoNodes) {
TEST(BadDataNodeTest, NNEventCallback) {
auto file_info = std::make_shared<struct FileInfo>();
+ file_info->file_length_ = 1; //To avoid running into EOF
file_info->blocks_.push_back(LocatedBlockProto());
LocatedBlockProto & block = file_info->blocks_[0];
ExtendedBlockProto *b = block.mutable_b();
@@ -215,6 +217,7 @@ TEST(BadDataNodeTest, NNEventCallback) {
TEST(BadDataNodeTest, RecoverableError) {
auto file_info = std::make_shared<struct FileInfo>();
+ file_info->file_length_ = 1; //To avoid running into EOF
file_info->blocks_.push_back(LocatedBlockProto());
LocatedBlockProto & block = file_info->blocks_[0];
ExtendedBlockProto *b = block.mutable_b();
@@ -265,6 +268,7 @@ TEST(BadDataNodeTest, RecoverableError) {
TEST(BadDataNodeTest, InternalError) {
auto file_info = std::make_shared<struct FileInfo>();
+ file_info->file_length_ = 1; //To avoid running into EOF
file_info->blocks_.push_back(LocatedBlockProto());
LocatedBlockProto & block = file_info->blocks_[0];
ExtendedBlockProto *b = block.mutable_b();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649aff11/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc
index f154be8..e28e4e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc
@@ -16,8 +16,6 @@
* limitations under the License.
*/
-//#include "expect.h"
-
#include "hdfspp_mini_dfs.h"
#include "hdfspp/hdfs_ext.h"
#include <chrono>
@@ -293,6 +291,41 @@ TEST_F(HdfsExtTest, TestChmodChown) {
hdfsFreeFileInfo(file_info, 1);
}
+//Testing EOF
+TEST_F(HdfsExtTest, TestEOF) {
+ HdfsHandle connection = cluster.connect_c();
+ hdfsFS fs = connection.handle();
+ EXPECT_NE(nullptr, fs);
+
+ //Write to a file
+ errno = 0;
+ int size = 256;
+ std::string path = "/eofTest";
+ hdfsFile file = hdfsOpenFile(fs, path.c_str(), O_WRONLY, 0, 0, 0);
+ EXPECT_NE(nullptr, file);
+ void * buf = malloc(size);
+ memset(buf, ' ', size);
+ EXPECT_EQ(size, hdfsWrite(fs, file, buf, size));
+ free(buf);
+ EXPECT_EQ(0, hdfsCloseFile(fs, file));
+ EXPECT_EQ(0, errno);
+
+ //Test normal reading (no EOF)
+ char buffer[300];
+ EXPECT_EQ(0, errno);
+ file = hdfsOpenFile(fs, path.c_str(), O_RDONLY, 0, 0, 0);
+ EXPECT_EQ(size, hdfsPread(fs, file, 0, buffer, sizeof(buffer)));
+ //Read executes correctly, but causes a warning (captured in HDFS-10595)
+ //and sets errno to EINPROGRESS 115 : Operation now in progress
+ errno = 0;
+
+ //Test reading at offset past the EOF
+ EXPECT_EQ(-1, hdfsPread(fs, file, sizeof(buffer), buffer, sizeof(buffer)));
+ EXPECT_EQ(Status::kInvalidOffset, errno);
+
+ EXPECT_EQ(0, hdfsCloseFile(fs, file));
+ EXPECT_EQ(0, errno);
+}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/52] [abbrv] hadoop git commit: libhdfs++: Get rid of lock in
RpcConnectionImpl destructor. Contributed by James Clampffer
Posted by jh...@apache.org.
libhdfs++: Get rid of lock in RpcConnectionImpl destructor. Contributed by James Clampffer
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6dd47cae
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6dd47cae
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6dd47cae
Branch: refs/heads/HDFS-8707
Commit: 6dd47cae8655f92ab2ae2729c0c08738db598405
Parents: 7ebecae
Author: James <jh...@apache.org>
Authored: Fri Oct 14 10:13:24 2016 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../native/libhdfspp/lib/rpc/rpc_connection.h | 75 ++++++++++----------
.../native/libhdfspp/tests/rpc_engine_test.cc | 6 +-
2 files changed, 41 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dd47cae/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
index 869be40..a6a07c4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
@@ -34,9 +34,11 @@
namespace hdfs {
-template <class NextLayer>
+template <class Socket>
class RpcConnectionImpl : public RpcConnection {
public:
+ MEMCHECKED_CLASS(RpcConnectionImpl);
+
RpcConnectionImpl(RpcEngine *engine);
virtual ~RpcConnectionImpl() override;
@@ -55,7 +57,7 @@ public:
virtual void FlushPendingRequests() override;
- NextLayer &next_layer() { return next_layer_; }
+ Socket &TEST_get_mutable_socket() { return socket_; }
void TEST_set_connected(bool connected) { connected_ = connected ? kConnected : kNotYetConnected; }
@@ -63,35 +65,34 @@ public:
const Options options_;
::asio::ip::tcp::endpoint current_endpoint_;
std::vector<::asio::ip::tcp::endpoint> additional_endpoints_;
- NextLayer next_layer_;
+ Socket socket_;
::asio::deadline_timer connect_timer_;
void ConnectComplete(const ::asio::error_code &ec, const ::asio::ip::tcp::endpoint &remote);
};
-template <class NextLayer>
-RpcConnectionImpl<NextLayer>::RpcConnectionImpl(RpcEngine *engine)
+template <class Socket>
+RpcConnectionImpl<Socket>::RpcConnectionImpl(RpcEngine *engine)
: RpcConnection(engine),
options_(engine->options()),
- next_layer_(engine->io_service()),
+ socket_(engine->io_service()),
connect_timer_(engine->io_service())
{
LOG_TRACE(kRPC, << "RpcConnectionImpl::RpcConnectionImpl called &" << (void*)this);
}
-template <class NextLayer>
-RpcConnectionImpl<NextLayer>::~RpcConnectionImpl() {
+template <class Socket>
+RpcConnectionImpl<Socket>::~RpcConnectionImpl() {
LOG_DEBUG(kRPC, << "RpcConnectionImpl::~RpcConnectionImpl called &" << (void*)this);
- std::lock_guard<std::mutex> state_lock(connection_state_lock_);
if (pending_requests_.size() > 0)
LOG_WARN(kRPC, << "RpcConnectionImpl::~RpcConnectionImpl called with items in the pending queue");
if (requests_on_fly_.size() > 0)
LOG_WARN(kRPC, << "RpcConnectionImpl::~RpcConnectionImpl called with items in the requests_on_fly queue");
}
-template <class NextLayer>
-void RpcConnectionImpl<NextLayer>::Connect(
+template <class Socket>
+void RpcConnectionImpl<Socket>::Connect(
const std::vector<::asio::ip::tcp::endpoint> &server,
const AuthInfo & auth_info,
RpcCallback &handler) {
@@ -109,8 +110,8 @@ void RpcConnectionImpl<NextLayer>::Connect(
this->ConnectAndFlush(server); // need "this" so compiler can infer type of CAF
}
-template <class NextLayer>
-void RpcConnectionImpl<NextLayer>::ConnectAndFlush(
+template <class Socket>
+void RpcConnectionImpl<Socket>::ConnectAndFlush(
const std::vector<::asio::ip::tcp::endpoint> &server) {
LOG_INFO(kRPC, << "ConnectAndFlush called");
@@ -139,7 +140,7 @@ void RpcConnectionImpl<NextLayer>::ConnectAndFlush(
current_endpoint_ = first_endpoint;
auto shared_this = shared_from_this();
- next_layer_.async_connect(first_endpoint, [shared_this, this, first_endpoint](const ::asio::error_code &ec) {
+ socket_.async_connect(first_endpoint, [shared_this, this, first_endpoint](const ::asio::error_code &ec) {
ConnectComplete(ec, first_endpoint);
});
@@ -155,9 +156,9 @@ void RpcConnectionImpl<NextLayer>::ConnectAndFlush(
});
}
-template <class NextLayer>
-void RpcConnectionImpl<NextLayer>::ConnectComplete(const ::asio::error_code &ec, const ::asio::ip::tcp::endpoint & remote) {
- auto shared_this = RpcConnectionImpl<NextLayer>::shared_from_this();
+template <class Socket>
+void RpcConnectionImpl<Socket>::ConnectComplete(const ::asio::error_code &ec, const ::asio::ip::tcp::endpoint & remote) {
+ auto shared_this = RpcConnectionImpl<Socket>::shared_from_this();
std::lock_guard<std::mutex> state_lock(connection_state_lock_);
connect_timer_.cancel();
@@ -190,7 +191,7 @@ void RpcConnectionImpl<NextLayer>::ConnectComplete(const ::asio::error_code &ec,
});
} else {
LOG_DEBUG(kRPC, << "Rpc connection failed; err=" << status.ToString());;
- std::string err = SafeDisconnect(get_asio_socket_ptr(&next_layer_));
+ std::string err = SafeDisconnect(get_asio_socket_ptr(&socket_));
if(!err.empty()) {
LOG_INFO(kRPC, << "Rpc connection failed to connect to endpoint, error closing connection: " << err);
}
@@ -202,7 +203,7 @@ void RpcConnectionImpl<NextLayer>::ConnectComplete(const ::asio::error_code &ec,
additional_endpoints_.erase(additional_endpoints_.begin());
current_endpoint_ = next_endpoint;
- next_layer_.async_connect(next_endpoint, [shared_this, this, next_endpoint](const ::asio::error_code &ec) {
+ socket_.async_connect(next_endpoint, [shared_this, this, next_endpoint](const ::asio::error_code &ec) {
ConnectComplete(ec, next_endpoint);
});
connect_timer_.expires_from_now(
@@ -219,8 +220,8 @@ void RpcConnectionImpl<NextLayer>::ConnectComplete(const ::asio::error_code &ec,
}
}
-template <class NextLayer>
-void RpcConnectionImpl<NextLayer>::SendHandshake(RpcCallback &handler) {
+template <class Socket>
+void RpcConnectionImpl<Socket>::SendHandshake(RpcCallback &handler) {
assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
LOG_TRACE(kRPC, << "RpcConnectionImpl::SendHandshake called");
@@ -228,7 +229,7 @@ void RpcConnectionImpl<NextLayer>::SendHandshake(RpcCallback &handler) {
auto shared_this = shared_from_this();
auto handshake_packet = PrepareHandshakePacket();
- ::asio::async_write(next_layer_, asio::buffer(*handshake_packet),
+ ::asio::async_write(socket_, asio::buffer(*handshake_packet),
[handshake_packet, handler, shared_this, this](
const ::asio::error_code &ec, size_t) {
Status status = ToStatus(ec);
@@ -236,15 +237,15 @@ void RpcConnectionImpl<NextLayer>::SendHandshake(RpcCallback &handler) {
});
}
-template <class NextLayer>
-void RpcConnectionImpl<NextLayer>::SendContext(RpcCallback &handler) {
+template <class Socket>
+void RpcConnectionImpl<Socket>::SendContext(RpcCallback &handler) {
assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
LOG_TRACE(kRPC, << "RpcConnectionImpl::SendContext called");
auto shared_this = shared_from_this();
auto context_packet = PrepareContextPacket();
- ::asio::async_write(next_layer_, asio::buffer(*context_packet),
+ ::asio::async_write(socket_, asio::buffer(*context_packet),
[context_packet, handler, shared_this, this](
const ::asio::error_code &ec, size_t) {
Status status = ToStatus(ec);
@@ -252,8 +253,8 @@ void RpcConnectionImpl<NextLayer>::SendContext(RpcCallback &handler) {
});
}
-template <class NextLayer>
-void RpcConnectionImpl<NextLayer>::OnSendCompleted(const ::asio::error_code &ec,
+template <class Socket>
+void RpcConnectionImpl<Socket>::OnSendCompleted(const ::asio::error_code &ec,
size_t) {
using std::placeholders::_1;
using std::placeholders::_2;
@@ -271,8 +272,8 @@ void RpcConnectionImpl<NextLayer>::OnSendCompleted(const ::asio::error_code &ec,
FlushPendingRequests();
}
-template <class NextLayer>
-void RpcConnectionImpl<NextLayer>::FlushPendingRequests() {
+template <class Socket>
+void RpcConnectionImpl<Socket>::FlushPendingRequests() {
using namespace ::std::placeholders;
// Lock should be held
@@ -335,7 +336,7 @@ void RpcConnectionImpl<NextLayer>::FlushPendingRequests() {
this->HandleRpcTimeout(timeout_req, ec);
});
- asio::async_write(next_layer_, asio::buffer(*payload),
+ asio::async_write(socket_, asio::buffer(*payload),
[shared_this, this, payload](const ::asio::error_code &ec,
size_t size) {
OnSendCompleted(ec, size);
@@ -352,8 +353,8 @@ void RpcConnectionImpl<NextLayer>::FlushPendingRequests() {
}
-template <class NextLayer>
-void RpcConnectionImpl<NextLayer>::OnRecvCompleted(const ::asio::error_code &original_ec,
+template <class Socket>
+void RpcConnectionImpl<Socket>::OnRecvCompleted(const ::asio::error_code &original_ec,
size_t) {
using std::placeholders::_1;
using std::placeholders::_2;
@@ -396,7 +397,7 @@ void RpcConnectionImpl<NextLayer>::OnRecvCompleted(const ::asio::error_code &ori
auto buf = ::asio::buffer(reinterpret_cast<char *>(¤t_response_state_->length_),
sizeof(current_response_state_->length_));
asio::async_read(
- next_layer_, buf,
+ socket_, buf,
[shared_this, this](const ::asio::error_code &ec, size_t size) {
OnRecvCompleted(ec, size);
});
@@ -405,7 +406,7 @@ void RpcConnectionImpl<NextLayer>::OnRecvCompleted(const ::asio::error_code &ori
current_response_state_->length_ = ntohl(current_response_state_->length_);
current_response_state_->data_.resize(current_response_state_->length_);
asio::async_read(
- next_layer_, ::asio::buffer(current_response_state_->data_),
+ socket_, ::asio::buffer(current_response_state_->data_),
[shared_this, this](const ::asio::error_code &ec, size_t size) {
OnRecvCompleted(ec, size);
});
@@ -425,8 +426,8 @@ void RpcConnectionImpl<NextLayer>::OnRecvCompleted(const ::asio::error_code &ori
}
}
-template <class NextLayer>
-void RpcConnectionImpl<NextLayer>::Disconnect() {
+template <class Socket>
+void RpcConnectionImpl<Socket>::Disconnect() {
assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
LOG_INFO(kRPC, << "RpcConnectionImpl::Disconnect called");
@@ -434,7 +435,7 @@ void RpcConnectionImpl<NextLayer>::Disconnect() {
request_over_the_wire_.reset();
if (connected_ == kConnecting || connected_ == kHandshaking || connected_ == kAuthenticating || connected_ == kConnected) {
// Don't print out errors, we were expecting a disconnect here
- SafeDisconnect(get_asio_socket_ptr(&next_layer_));
+ SafeDisconnect(get_asio_socket_ptr(&socket_));
}
connected_ = kDisconnected;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dd47cae/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
index 3e8c93f..08218f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
@@ -118,7 +118,7 @@ TEST(RpcEngineTest, TestRoundTrip) {
RpcResponseHeaderProto h;
h.set_callid(1);
h.set_status(RpcResponseHeaderProto::SUCCESS);
- EXPECT_CALL(conn->next_layer(), Produce())
+ EXPECT_CALL(conn->TEST_get_mutable_socket(), Produce())
.WillOnce(Return(RpcResponse(h, server_resp.SerializeAsString())));
std::shared_ptr<RpcConnection> conn_ptr(conn);
@@ -153,7 +153,7 @@ TEST(RpcEngineTest, TestConnectionResetAndFail) {
RpcResponseHeaderProto h;
h.set_callid(1);
h.set_status(RpcResponseHeaderProto::SUCCESS);
- EXPECT_CALL(conn->next_layer(), Produce())
+ EXPECT_CALL(conn->TEST_get_mutable_socket(), Produce())
.WillOnce(Return(RpcResponse(
h, "", make_error_code(::asio::error::connection_reset))));
@@ -455,7 +455,7 @@ TEST(RpcEngineTest, TestTimeout) {
conn->TEST_set_connected(true);
conn->StartReading();
- EXPECT_CALL(conn->next_layer(), Produce())
+ EXPECT_CALL(conn->TEST_get_mutable_socket(), Produce())
.WillOnce(Return(std::make_pair(::asio::error::would_block, "")));
std::shared_ptr<RpcConnection> conn_ptr(conn);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/52] [abbrv] hadoop git commit: HDFS-11027: libbhdfs++: Don't
retry if there is an authentication failure. Contributed by Xiaowei Zhu.
Posted by jh...@apache.org.
HDFS-11027: libbhdfs++: Don't retry if there is an authentication failure. Contributed by Xiaowei Zhu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b7d9a0f8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b7d9a0f8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b7d9a0f8
Branch: refs/heads/HDFS-8707
Commit: b7d9a0f827c0688fa1ebe6b71f0e0a4839d73ea2
Parents: 59a3926
Author: James <jh...@apache.org>
Authored: Mon Oct 24 10:47:15 2016 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../src/main/native/libhdfspp/include/hdfspp/status.h | 3 +++
.../src/main/native/libhdfspp/lib/common/status.cc | 13 ++++++++++++-
.../src/main/native/libhdfspp/lib/rpc/rpc_engine.cc | 4 +++-
3 files changed, 18 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7d9a0f8/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
index ffbb2bf..39cf8c6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
@@ -62,6 +62,9 @@ class Status {
// get error code
int code() const { return code_; }
+ // if retry can possibly recover an error
+ bool notWorthRetry() const;
+
enum Code {
kOk = 0,
kInvalidArgument = static_cast<unsigned>(std::errc::invalid_argument),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7d9a0f8/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
index 540f8c1..15d304a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
@@ -22,6 +22,7 @@
#include <sstream>
#include <cstring>
#include <map>
+#include <set>
namespace hdfs {
@@ -49,6 +50,12 @@ const static std::map<std::string, int> kKnownServerExceptionClasses = {
{kPathIsNotEmptyDirectoryException, Status::kPathIsNotEmptyDirectory}
};
+// Errors that retry cannot fix. TODO: complete the list.
+const static std::set<int> noRetryExceptions = {
+ Status::kPermissionDenied,
+ Status::kAuthenticationFailed,
+ Status::kAccessControlException
+};
Status::Status(int code, const char *msg1)
: code_(code) {
@@ -120,7 +127,7 @@ Status Status::Exception(const char *exception_class_name, const char *error_mes
}
Status Status::Error(const char *error_message) {
- return Status(kAuthenticationFailed, error_message);
+ return Exception("Exception", error_message);
}
Status Status::AuthenticationFailed() {
@@ -147,4 +154,8 @@ std::string Status::ToString() const {
return ss.str();
}
+bool Status::notWorthRetry() const {
+ return noRetryExceptions.find(code_) != noRetryExceptions.end();
+}
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7d9a0f8/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
index 72a0c55..7c280b8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
@@ -322,7 +322,9 @@ void RpcEngine::RpcCommsError(
RetryAction retry = RetryAction::fail(""); // Default to fail
- if (retry_policy()) {
+ if (status.notWorthRetry()) {
+ retry = RetryAction::fail(status.ToString().c_str());
+ } else if (retry_policy()) {
retry = retry_policy()->ShouldRetry(status, req->IncrementRetryCount(), req->get_failover_count(), true);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/52] [abbrv] hadoop git commit: HDFS-10441: libhdfs++: HA namenode
support. Contributed by James Clampffer.
Posted by jh...@apache.org.
HDFS-10441: libhdfs++: HA namenode support. Contributed by James Clampffer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4cb0dad5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4cb0dad5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4cb0dad5
Branch: refs/heads/HDFS-8707
Commit: 4cb0dad5e535f4ee79c32a54f8958bf7d4402123
Parents: d75c835
Author: James <jh...@apache.org>
Authored: Fri Jul 29 20:10:53 2016 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../main/native/libhdfspp/examples/cat/c/cat.c | 1 +
.../native/libhdfspp/include/hdfspp/events.h | 9 +-
.../native/libhdfspp/include/hdfspp/options.h | 42 +++-
.../native/libhdfspp/include/hdfspp/status.h | 24 +-
.../native/libhdfspp/lib/common/CMakeLists.txt | 2 +-
.../libhdfspp/lib/common/hdfs_configuration.cc | 123 ++++++++++
.../libhdfspp/lib/common/hdfs_configuration.h | 5 +
.../main/native/libhdfspp/lib/common/logging.cc | 2 +-
.../main/native/libhdfspp/lib/common/logging.h | 9 +-
.../libhdfspp/lib/common/namenode_info.cc | 132 ++++++++++
.../native/libhdfspp/lib/common/namenode_info.h | 49 ++++
.../main/native/libhdfspp/lib/common/options.cc | 21 +-
.../native/libhdfspp/lib/common/retry_policy.cc | 43 +++-
.../native/libhdfspp/lib/common/retry_policy.h | 83 ++++++-
.../main/native/libhdfspp/lib/common/status.cc | 73 ++++--
.../src/main/native/libhdfspp/lib/common/uri.cc | 48 +++-
.../src/main/native/libhdfspp/lib/common/uri.h | 6 +-
.../main/native/libhdfspp/lib/fs/filesystem.cc | 39 ++-
.../libhdfspp/lib/fs/namenode_operations.cc | 18 +-
.../libhdfspp/lib/fs/namenode_operations.h | 7 +-
.../native/libhdfspp/lib/rpc/rpc_connection.cc | 56 ++++-
.../native/libhdfspp/lib/rpc/rpc_connection.h | 13 +-
.../main/native/libhdfspp/lib/rpc/rpc_engine.cc | 241 ++++++++++++++++---
.../main/native/libhdfspp/lib/rpc/rpc_engine.h | 75 +++++-
.../native/libhdfspp/tests/rpc_engine_test.cc | 24 +-
25 files changed, 1016 insertions(+), 129 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cat/c/cat.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cat/c/cat.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cat/c/cat.c
index 5839308..dec8758 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cat/c/cat.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cat/c/cat.c
@@ -86,6 +86,7 @@ void parse_uri(const char * uri_string, struct Uri * uri) {
};
int main(int argc, char** argv) {
+
char error_text[1024];
if (argc != 2) {
fprintf(stderr, "usage: cat [hdfs://[<hostname>:<port>]]/<path-to-file>\n");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/events.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/events.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/events.h
index 43187a5..80c3712 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/events.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/events.h
@@ -29,13 +29,20 @@ namespace hdfs {
* Supported event names. These names will stay consistent in libhdfs callbacks.
*
* Other events not listed here may be seen, but they are not stable and
- * should not be counted on.
+ * should not be counted on. May need to be broken up into more components
+ * as more events are added.
*/
static constexpr const char * FS_NN_CONNECT_EVENT = "NN::connect";
static constexpr const char * FS_NN_READ_EVENT = "NN::read";
static constexpr const char * FS_NN_WRITE_EVENT = "NN::write";
+// NN failover event due to issues with the current NN; might be standby, might be dead.
+// Invokes the fs_event_callback using the nameservice name in the cluster string.
+// The uint64_t value argument holds an address that can be reinterpreted as a const char *
+// and provides the full URI of the node the failover will attempt to connect to next.
+static constexpr const char * FS_NN_FAILOVER_EVENT = "NN::failover";
+
static constexpr const char * FILE_DN_CONNECT_EVENT = "DN::connect";
static constexpr const char * FILE_DN_READ_EVENT = "DN::read";
static constexpr const char * FILE_DN_WRITE_EVENT = "DN::write";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
index 8562f6d..1acfe1a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
@@ -20,8 +20,29 @@
#include "common/uri.h"
+#include <string>
+#include <vector>
+#include <map>
+
namespace hdfs {
+
+struct NamenodeInfo {
+ NamenodeInfo(const std::string &nameservice, const std::string &nodename, const URI &uri) :
+ nameservice(nameservice), name(nodename), uri(uri) {}
+ NamenodeInfo(){};
+ //nameservice this belongs to
+ std::string nameservice;
+ //node name
+ std::string name;
+ //host:port
+ URI uri;
+
+ //get server hostname and port (aka service)
+ std::string get_host() const;
+ std::string get_port() const;
+};
+
/**
* Options to control the behavior of the libhdfspp library.
**/
@@ -44,7 +65,7 @@ struct Options {
* Maximum number of retries for RPC operations
**/
int max_rpc_retries;
- static const int kNoRetry = -1;
+ static const int kNoRetry = 0;
static const int kDefaultMaxRpcRetries = kNoRetry;
/**
@@ -66,6 +87,25 @@ struct Options {
URI defaultFS;
/**
+ * Namenodes used to provide HA for this cluster if applicable
+ **/
+ std::map<std::string, std::vector<NamenodeInfo>> services;
+
+
+ /**
+ * Client failover attempts before failover gives up
+ **/
+ int failover_max_retries;
+ static const unsigned int kDefaultFailoverMaxRetries = 15;
+
+ /**
+ * Client failover attempts before failover gives up if server
+ * connection is timing out.
+ **/
+ int failover_connection_max_retries;
+ static const unsigned int kDefaultFailoverConnectionMaxRetries = 0;
+
+ /*
* Which form of authentication to use with the server
* Default: simple
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
index 1628d8c..f217cad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
@@ -27,15 +27,20 @@ class Status {
public:
// Create a success status.
Status() : code_(0) {};
+
+ // Note: Avoid calling the Status constructors directly, call the factory methods instead
+
+ // Used for common status types
Status(int code, const char *msg);
- Status(int code, const char *msg1, const char *msg2);
+ // Used for server side exceptions reported through RpcResponseProto and similar
+ Status(int code, const char *exception_class, const char *exception_details);
// Factory methods
static Status OK();
static Status InvalidArgument(const char *msg);
static Status ResourceUnavailable(const char *msg);
static Status Unimplemented();
- static Status Exception(const char *expception_class_name, const char *error_message);
+ static Status Exception(const char *exception_class_name, const char *exception_details);
static Status Error(const char *error_message);
static Status AuthenticationFailed();
static Status Canceled();
@@ -61,13 +66,28 @@ class Status {
kNotADirectory = static_cast<unsigned>(std::errc::not_a_directory),
kFileAlreadyExists = static_cast<unsigned>(std::errc::file_exists),
kPathIsNotEmptyDirectory = static_cast<unsigned>(std::errc::directory_not_empty),
+
+ // non-errc codes start at 256
kException = 256,
kAuthenticationFailed = 257,
+ kAccessControlException = 258,
+ kStandbyException = 259,
+ kSnapshotProtocolException = 260,
};
+ std::string get_exception_class_str() const {
+ return exception_class_;
+ }
+
+ int get_server_exception_type() const {
+ return code_;
+ }
+
private:
int code_;
std::string msg_;
+
+ std::string exception_class_;
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
index 484d1a4..501d3b5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
@@ -19,6 +19,6 @@ if(NEED_LINK_DL)
set(LIB_DL dl)
endif()
-add_library(common_obj OBJECT base64.cc status.cc sasl_digest_md5.cc hdfs_public_api.cc options.cc configuration.cc configuration_loader.cc hdfs_configuration.cc uri.cc util.cc retry_policy.cc cancel_tracker.cc logging.cc libhdfs_events_impl.cc auth_info.cc)
+add_library(common_obj OBJECT base64.cc status.cc sasl_digest_md5.cc hdfs_public_api.cc options.cc configuration.cc configuration_loader.cc hdfs_configuration.cc uri.cc util.cc retry_policy.cc cancel_tracker.cc logging.cc libhdfs_events_impl.cc auth_info.cc namenode_info.cc)
add_library(common $<TARGET_OBJECTS:common_obj> $<TARGET_OBJECTS:uriparser2_obj>)
target_link_libraries(common ${LIB_DL})
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc
index ef67af9..70775b8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc
@@ -17,6 +17,13 @@
*/
#include "common/hdfs_configuration.h"
+#include "common/logging.h"
+
+#include <exception>
+
+#ifndef DEFAULT_SCHEME
+ #define DEFAULT_SCHEME "hdfs://"
+#endif
namespace hdfs {
@@ -40,6 +47,94 @@ void OptionalSet(T& target, optional<U> value) {
target = *value;
}
+std::vector<std::string> SplitOnComma(const std::string &s, bool include_empty_strings) {
+ std::vector<std::string> res;
+ std::string buf;
+
+ for(unsigned int i=0;i<s.size();i++) {
+ char c = s[i];
+ if(c != ',') {
+ buf += c;
+ } else {
+ if(!include_empty_strings && buf.empty()) {
+ // Skip adding empty strings if needed
+ continue;
+ }
+ res.push_back(buf);
+ buf.clear();
+ }
+ }
+
+ if(buf.size() > 0)
+ res.push_back(buf);
+
+ return res;
+}
+
+// Prepend hdfs:// to string if there isn't already a scheme
+// Converts unset optional into empty string
+std::string PrependHdfsScheme(optional<std::string> str) {
+ if(!str)
+ return "";
+
+ if(str.value().find("://") == std::string::npos)
+ return DEFAULT_SCHEME + str.value();
+ return str.value();
+}
+
+// It's either use this, goto, or a lot of returns w/ status checks
+struct ha_parse_error : public std::exception {
+ std::string desc;
+ ha_parse_error(const std::string &val) : desc(val) {};
+ const char *what() const noexcept override {
+ return desc.c_str();
+ };
+};
+
+std::vector<NamenodeInfo> HdfsConfiguration::LookupNameService(const std::string &nameservice) {
+ std::vector<NamenodeInfo> namenodes;
+ try {
+ // Find namenodes that belong to nameservice
+ std::vector<std::string> namenode_ids;
+ {
+ std::string service_nodes = std::string("dfs.ha.namenodes.") + nameservice;
+ optional<std::string> namenode_list = Get(service_nodes);
+ if(namenode_list)
+ namenode_ids = SplitOnComma(namenode_list.value(), false);
+ else
+ throw ha_parse_error("unable to find " + service_nodes);
+
+ for(auto it=namenode_ids.begin(); it != namenode_ids.end(); it++)
+ LOG_INFO(kRPC, << "Namenode: " << *it);
+ }
+
+ // should this error if we only find 1 NN?
+ if(namenode_ids.empty())
+ throw ha_parse_error("No namenodes found for nameservice " + nameservice);
+
+ // Get URI for each HA namenode
+ for(auto node_id=namenode_ids.begin(); node_id != namenode_ids.end(); node_id++) {
+ // find URI
+ std::string dom_node_name = std::string("dfs.namenode.rpc-address.") + nameservice + "." + *node_id;
+ optional<URI> node_uri = URI::parse_from_string(PrependHdfsScheme(Get(dom_node_name)));
+
+ if(!node_uri) {
+ throw ha_parse_error("unable to find " + dom_node_name);
+ }
+
+ URI uri = node_uri.value();
+ LOG_INFO(kRPC, << "Read the following HA Namenode URI from config" << uri.GetDebugString());
+
+ NamenodeInfo node(nameservice, *node_id, uri);
+ namenodes.push_back(node);
+ }
+ } catch (ha_parse_error e) {
+ LOG_ERROR(kRPC, << "HA cluster detected but failed because : " << e.what());
+ namenodes.clear(); // Don't return inconsistent view
+ }
+ return namenodes;
+}
+
// Interprets the resources to build an Options object
Options HdfsConfiguration::GetOptions() {
Options result;
@@ -50,7 +145,35 @@ Options HdfsConfiguration::GetOptions() {
OptionalSet(result.rpc_retry_delay_ms, GetInt(kIpcClientConnectRetryIntervalKey));
OptionalSet(result.defaultFS, GetUri(kFsDefaultFsKey));
+
+ OptionalSet(result.failover_max_retries, GetInt(kDfsClientFailoverMaxAttempts));
+ OptionalSet(result.failover_connection_max_retries, GetInt(kDfsClientFailoverConnectionRetriesOnTimeouts));
+
+ // Load all nameservices if it's HA configured
+ optional<std::string> dfs_nameservices = Get("dfs.nameservices");
+ if(dfs_nameservices) {
+ std::string nameservice = dfs_nameservices.value();
+
+ std::vector<std::string> all_services = SplitOnComma(nameservice, false);
+
+ // Look up nodes for each nameservice so that FileSystem object can support
+ // multiple nameservices by ID.
+ for(const std::string &service : all_services) {
+ if(service.empty())
+ continue;
+
+ LOG_DEBUG(kFileSystem, << "Parsing info for nameservice: " << service);
+ std::vector<NamenodeInfo> nodes = LookupNameService(service);
+ if(nodes.empty()) {
+ LOG_WARN(kFileSystem, << "Nameservice \"" << service << "\" declared in config but nodes aren't");
+ } else {
+ result.services[service] = nodes;
+ }
+ }
+ }
+
optional<std::string> authentication_value = Get(kHadoopSecurityAuthenticationKey);
+
if (authentication_value ) {
std::string fixed_case_value = fixCase(authentication_value.value());
if (fixed_case_value == fixCase(kHadoopSecurityAuthentication_kerberos))
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.h
index c6ead66..459364f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.h
@@ -46,6 +46,10 @@ class HdfsConfiguration : public Configuration {
static constexpr const char * kHadoopSecurityAuthentication_simple = "simple";
static constexpr const char * kHadoopSecurityAuthentication_kerberos = "kerberos";
+ static constexpr const char * kDfsClientFailoverMaxAttempts = "dfs.client.failover.max.attempts";
+ static constexpr const char * kDfsClientFailoverConnectionRetriesOnTimeouts = "dfs.client.failover.connection.retries.on.timeouts";
+
+
private:
friend class ConfigurationLoader;
@@ -57,6 +61,7 @@ private:
HdfsConfiguration(const ConfigMap &src_map);
static std::vector<std::string> GetDefaultFilenames();
+ std::vector<NamenodeInfo> LookupNameService(const std::string &nameservice);
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.cc
index 39ed944..ac1c336 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.cc
@@ -177,12 +177,12 @@ LogMessage& LogMessage::operator<<(uint64_t val) {
return *this;
}
-
LogMessage& LogMessage::operator<<(void *ptr) {
msg_buffer_ << ptr;
return *this;
}
+
std::string LogMessage::MsgString() const {
return msg_buffer_.str();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h
index 9dc0c5f..f807fc4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h
@@ -19,6 +19,8 @@
#ifndef LIB_COMMON_LOGGING_H_
#define LIB_COMMON_LOGGING_H_
+#include <asio/ip/tcp.hpp>
+
#include "hdfspp/log.h"
#include <iostream>
@@ -179,11 +181,10 @@ class LogMessage {
LogMessage& operator<<(const std::string*);
LogMessage& operator<<(const std::string&);
- LogMessage& operator<<(const ::asio::ip::tcp::endpoint& endpoint);
-
//convert to a string "true"/"false"
LogMessage& operator<<(bool);
+ //integral types
LogMessage& operator<<(int32_t);
LogMessage& operator<<(uint32_t);
LogMessage& operator<<(int64_t);
@@ -192,6 +193,10 @@ class LogMessage {
//print address as hex
LogMessage& operator<<(void *);
+ //asio types
+ LogMessage& operator<<(const ::asio::ip::tcp::endpoint& endpoint);
+
+
std::string MsgString() const;
private:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc
new file mode 100644
index 0000000..bc38be7
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "namenode_info.h"
+
+#include "common/continuation/asio.h"
+#include "common/logging.h"
+
+#include <sstream>
+#include <utility>
+#include <future>
+#include <memory>
+
+namespace hdfs {
+
+ResolvedNamenodeInfo& ResolvedNamenodeInfo::operator=(const NamenodeInfo &info) {
+ nameservice = info.nameservice;
+ name = info.name;
+ uri = info.uri;
+ return *this;
+}
+
+
+
+std::string ResolvedNamenodeInfo::str() const {
+ std::stringstream ss;
+ ss << "ResolvedNamenodeInfo {nameservice: " << nameservice << ", name: " << name << ", uri: " << uri.str();
+ ss << ", host: " << uri.get_host();
+ auto port = uri.get_port();
+ if(port)
+ ss << ", port: " << port.value();
+ else
+ ss << ", port: unable to parse";
+
+ ss << ", scheme: " << uri.get_scheme();
+
+ ss << " [";
+ for(unsigned int i=0;i<endpoints.size();i++)
+ ss << endpoints[i] << " ";
+ ss << "] }";
+
+ return ss.str();
+}
+
+
+bool ResolveInPlace(::asio::io_service *ioservice, ResolvedNamenodeInfo &info) {
+ // this isn't very memory friendly, but if it needs to be called often there are bigger issues at hand
+ info.endpoints.clear();
+ std::vector<ResolvedNamenodeInfo> resolved = BulkResolve(ioservice, {info});
+ if(resolved.size() != 1)
+ return false;
+
+ info.endpoints = resolved[0].endpoints;
+ if(info.endpoints.size() == 0)
+ return false;
+ return true;
+}
+
+
+std::vector<ResolvedNamenodeInfo> BulkResolve(::asio::io_service *ioservice, const std::vector<NamenodeInfo> &nodes) {
+ using namespace asio_continuation;
+
+ typedef std::vector<asio::ip::tcp::endpoint> endpoint_vector;
+ typedef Pipeline<endpoint_vector> resolve_pipeline_t;
+
+
+ std::vector<std::pair<resolve_pipeline_t*, std::shared_ptr<std::promise<Status>>>> pipelines;
+ pipelines.reserve(nodes.size());
+
+ std::vector<ResolvedNamenodeInfo> resolved_info;
+ // This must never reallocate once async ops begin
+ resolved_info.reserve(nodes.size());
+
+ for(unsigned int i=0; i<nodes.size(); i++) {
+ std::string host = nodes[i].get_host();
+ std::string port = nodes[i].get_port();
+
+ ResolvedNamenodeInfo resolved;
+ resolved = nodes[i];
+ resolved_info.push_back(resolved);
+
+ // build the pipeline
+ resolve_pipeline_t *pipeline = resolve_pipeline_t::Create();
+ auto resolve_step = Resolve(ioservice, host, port, std::back_inserter(pipeline->state()));
+ pipeline->Push(resolve_step);
+
+ // make a status associated with current pipeline
+ std::shared_ptr<std::promise<Status>> active_stat = std::make_shared<std::promise<Status>>();
+ pipelines.push_back(std::make_pair(pipeline, active_stat));
+
+ pipeline->Run([i,active_stat, &resolved_info](const Status &s, const endpoint_vector &ends){
+ resolved_info[i].endpoints = ends;
+ active_stat->set_value(s);
+ });
+
+ }
+
+ // Join all async operations
+ std::vector<ResolvedNamenodeInfo> return_set;
+ for(unsigned int i=0; i<pipelines.size();i++) {
+ std::shared_ptr<std::promise<Status>> promise = pipelines[i].second;
+
+ std::future<Status> future = promise->get_future();
+ Status stat = future.get();
+
+ // Clear endpoints if we hit an error
+ if(!stat.ok()) {
+ LOG_WARN(kRPC, << "Unable to resolve endpoints for " << nodes[i].uri.str());
+ resolved_info[i].endpoints.clear();
+ }
+ }
+
+ return resolved_info;
+}
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.h
new file mode 100644
index 0000000..fdee8d7
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.h
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef COMMON_HDFS_NAMENODE_INFO_H_
+#define COMMON_HDFS_NAMENODE_INFO_H_
+
+#include <asio.hpp>
+#include <hdfspp/options.h>
+
+#include <string>
+#include <vector>
+
+namespace hdfs {
+
+// Internal representation of namenode info that keeps track
+// of its endpoints.
+struct ResolvedNamenodeInfo : public NamenodeInfo {
+ ResolvedNamenodeInfo& operator=(const NamenodeInfo &info);
+ std::string str() const;
+
+ std::vector<::asio::ip::tcp::endpoint> endpoints;
+};
+
+// Clear endpoints if set and resolve all of them in parallel.
+// Only successful lookups will be placed in the result set.
+std::vector<ResolvedNamenodeInfo> BulkResolve(::asio::io_service *ioservice, const std::vector<NamenodeInfo> &nodes);
+
+// Clear endpoints, if any, and resolve them again
+// Return true if endpoints were resolved
+bool ResolveInPlace(::asio::io_service *ioservice, ResolvedNamenodeInfo &info);
+
+}
+
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc
index 305ea1a..29b45b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc
@@ -26,12 +26,27 @@ const int Options::kNoRetry;
const int Options::kDefaultMaxRpcRetries;
const int Options::kDefaultRpcRetryDelayMs;
const unsigned int Options::kDefaultHostExclusionDuration;
+const unsigned int Options::kDefaultFailoverMaxRetries;
+const unsigned int Options::kDefaultFailoverConnectionMaxRetries;
-Options::Options() : rpc_timeout(kDefaultRpcTimeout), rpc_connect_timeout(kDefaultRpcConnectTimeout),
+Options::Options() : rpc_timeout(kDefaultRpcTimeout),
+ rpc_connect_timeout(kDefaultRpcConnectTimeout),
max_rpc_retries(kDefaultMaxRpcRetries),
rpc_retry_delay_ms(kDefaultRpcRetryDelayMs),
host_exclusion_duration(kDefaultHostExclusionDuration),
defaultFS(),
- authentication(kDefaultAuthentication)
-{}
+ failover_max_retries(kDefaultFailoverMaxRetries),
+ failover_connection_max_retries(kDefaultFailoverConnectionMaxRetries),
+ authentication(kDefaultAuthentication) {}
+
+std::string NamenodeInfo::get_host() const {
+ return uri.get_host();
+}
+std::string NamenodeInfo::get_port() const {
+ optional<uint16_t> p = uri.get_port();
+ if(!p)
+ return std::to_string(-1);
+ return std::to_string(p.value());
+}
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.cc
index 1857e20..a885d53 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.cc
@@ -17,31 +17,64 @@
*/
#include "common/retry_policy.h"
+#include "common/logging.h"
+
+#include <sstream>
namespace hdfs {
RetryAction FixedDelayRetryPolicy::ShouldRetry(
const Status &s, uint64_t retries, uint64_t failovers,
bool isIdempotentOrAtMostOnce) const {
- (void)s;
+ LOG_TRACE(kRPC, << "FixedDelayRetryPolicy::ShouldRetry(retries=" << retries << ", failovers=" << failovers << ")");
(void)isIdempotentOrAtMostOnce;
if (retries + failovers >= max_retries_) {
return RetryAction::fail(
- "Failovers (" + std::to_string(retries + failovers) +
- ") exceeded maximum retries (" + std::to_string(max_retries_) + ")");
+ "Failovers and retries(" + std::to_string(retries + failovers) +
+ ") exceeded maximum retries (" + std::to_string(max_retries_) + "), Status: " +
+ s.ToString());
} else {
return RetryAction::retry(delay_);
}
}
+
RetryAction NoRetryPolicy::ShouldRetry(
const Status &s, uint64_t retries, uint64_t failovers,
bool isIdempotentOrAtMostOnce) const {
- (void)s;
+ LOG_TRACE(kRPC, << "NoRetryPolicy::ShouldRetry(retries=" << retries << ", failovers=" << failovers << ")");
(void)retries;
(void)failovers;
(void)isIdempotentOrAtMostOnce;
- return RetryAction::fail("No retry");
+ return RetryAction::fail("No retry, Status: " + s.ToString());
+}
+
+
+RetryAction FixedDelayWithFailover::ShouldRetry(const Status &s, uint64_t retries,
+ uint64_t failovers,
+ bool isIdempotentOrAtMostOnce) const {
+ (void)isIdempotentOrAtMostOnce;
+ LOG_TRACE(kRPC, << "FixedDelayWithFailover::ShouldRetry(retries=" << retries << ", failovers=" << failovers << ")");
+
+ if(s.code() == ::asio::error::timed_out && failovers < max_failover_retries_) {
+ // Try connecting to another NN in case this one keeps timing out
+ // Can add the backoff wait specified by dfs.client.failover.sleep.base.millis here
+ return RetryAction::failover(delay_);
+ }
+
+ if(retries < max_retries_) {
+ LOG_TRACE(kRPC, << "FixedDelayWithFailover::ShouldRetry: retries < max_retries_");
+ return RetryAction::retry(delay_);
+ } else if (retries >= max_retries_ && failovers < max_failover_retries_) {
+ LOG_TRACE(kRPC, << "FixedDelayWithFailover::ShouldRetry: retries >= max_retries_ && failovers < max_failover_retries_");
+ return RetryAction::failover(delay_);
+ } else if (retries >= max_retries_ && failovers == max_failover_retries_) {
+ LOG_TRACE(kRPC, << "FixedDelayWithFailover::ShouldRetry: retries >= max_retries_ && failovers == max_failover_retries_");
+ // 1 last retry on new connection
+ return RetryAction::retry(delay_);
+ }
+
+ return RetryAction::fail("Retry and failover didn't work, Status: " + s.ToString());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.h
index afa4f7d..0b5bd80 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.h
@@ -43,13 +43,31 @@ class RetryAction {
static RetryAction retry(uint64_t delay) {
return RetryAction(RETRY, delay, "");
}
- static RetryAction failover() {
- return RetryAction(FAILOVER_AND_RETRY, 0, "");
+ static RetryAction failover(uint64_t delay) {
+ return RetryAction(FAILOVER_AND_RETRY, delay, "");
}
+
+ std::string decision_str() const {
+ switch(action) {
+ case FAIL: return "FAIL";
+ case RETRY: return "RETRY";
+ case FAILOVER_AND_RETRY: return "FAILOVER_AND_RETRY";
+ default: return "UNDEFINED ACTION";
+ }
+ };
};
class RetryPolicy {
+ protected:
+ uint64_t delay_;
+ uint64_t max_retries_;
+ RetryPolicy(uint64_t delay, uint64_t max_retries) :
+ delay_(delay), max_retries_(max_retries) {}
+
public:
+ RetryPolicy() {};
+
+ virtual ~RetryPolicy() {}
/*
* If there was an error in communications, responds with the configured
* action to take.
@@ -58,23 +76,71 @@ class RetryPolicy {
uint64_t failovers,
bool isIdempotentOrAtMostOnce) const = 0;
- virtual ~RetryPolicy() {}
+ virtual std::string str() const { return "Base RetryPolicy"; }
};
+
+/*
+ * Overview of how the failover retry policy works:
+ *
+ * 1) Acts the same as FixedDelayRetryPolicy in terms of connection retries against a single NN
+ * with two differences:
+ * a) If we have retried more than the maximum number of retries we will failover to the
+ * other node and reset the retry counter rather than error out. It will begin the same
+ * routine on the other node.
+ * b) If an attempted connection times out and max_failover_conn_retries_ is less than the
+ * normal number of retries it will failover sooner. The connection timeout retry limit
+ * defaults to zero; the idea being that if a node is unresponsive it's better to just
+ * try the secondary rather than incur the timeout cost multiple times.
+ *
+ * 2) Keeps track of the failover count in the same way that the retry count is tracked. If failover
+ * is triggered more than a set number (dfs.client.failover.max.attempts) of times then the operation
+ * will error out in the same way that a non-HA operation would error if it ran out of retries.
+ *
+ * 3) Failover between namenodes isn't instantaneous so the RPC retry delay is reused to add a small
+ * delay between failover attempts. This helps prevent the client from quickly using up all of
+ * its failover attempts while thrashing between namenodes that are both temporarily marked standby.
+ * Note: The java client implements exponential backoff here with a base other than the rpc delay,
+ * and this will do the same here in the future. This doesn't do any sort of exponential backoff
+ * and the name can be changed to ExponentialDelayWithFailover when backoff is implemented.
+ */
+class FixedDelayWithFailover : public RetryPolicy {
+ public:
+ FixedDelayWithFailover(uint64_t delay, uint64_t max_retries,
+ uint64_t max_failover_retries,
+ uint64_t max_failover_conn_retries)
+ : RetryPolicy(delay, max_retries), max_failover_retries_(max_failover_retries),
+ max_failover_conn_retries_(max_failover_conn_retries) {}
+
+ RetryAction ShouldRetry(const Status &s, uint64_t retries,
+ uint64_t failovers,
+ bool isIdempotentOrAtMostOnce) const override;
+
+ std::string str() const override { return "FixedDelayWithFailover"; }
+
+ private:
+ // Attempts to fail over
+ uint64_t max_failover_retries_;
+ // Attempts to fail over if connection times out rather than
+ // tring to connect and wait for the timeout delay failover_retries_
+ // times.
+ uint64_t max_failover_conn_retries_;
+};
+
+
/*
* Returns a fixed delay up to a certain number of retries
*/
class FixedDelayRetryPolicy : public RetryPolicy {
public:
FixedDelayRetryPolicy(uint64_t delay, uint64_t max_retries)
- : delay_(delay), max_retries_(max_retries) {}
+ : RetryPolicy(delay, max_retries) {}
RetryAction ShouldRetry(const Status &s, uint64_t retries,
uint64_t failovers,
bool isIdempotentOrAtMostOnce) const override;
- private:
- uint64_t delay_;
- uint64_t max_retries_;
+
+ std::string str() const override { return "FixedDelayRetryPolicy"; }
};
/*
@@ -82,9 +148,12 @@ class FixedDelayRetryPolicy : public RetryPolicy {
*/
class NoRetryPolicy : public RetryPolicy {
public:
+ NoRetryPolicy() {};
RetryAction ShouldRetry(const Status &s, uint64_t retries,
uint64_t failovers,
bool isIdempotentOrAtMostOnce) const override;
+
+ std::string str() const override { return "NoRetryPolicy"; }
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
index d6c9875..796b1a2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
@@ -21,33 +21,55 @@
#include <cassert>
#include <sstream>
#include <cstring>
+#include <map>
namespace hdfs {
-const char * kStatusAccessControlException = "org.apache.hadoop.security.AccessControlException";
-const char * kStatusSaslException = "javax.security.sasl.SaslException";
-const char * kPathNotFoundException = "org.apache.hadoop.fs.InvalidPathException";
-const char * kPathNotFoundException2 = "java.io.FileNotFoundException";
-const char * kPathIsNotDirectoryException = "org.apache.hadoop.fs.PathIsNotDirectoryException";
-const char * kSnapshotException = "org.apache.hadoop.hdfs.protocol.SnapshotException";
-const char * kFileAlreadyExistsException = "org.apache.hadoop.fs.FileAlreadyExistsException";
+// Server side exceptions that we capture from the RpcResponseHeaderProto
+const char * kStatusAccessControlException = "org.apache.hadoop.security.AccessControlException";
+const char * kPathIsNotDirectoryException = "org.apache.hadoop.fs.PathIsNotDirectoryException";
+const char * kSnapshotException = "org.apache.hadoop.hdfs.protocol.SnapshotException";
+const char * kStatusStandbyException = "org.apache.hadoop.ipc.StandbyException";
+const char * kStatusSaslException = "javax.security.sasl.SaslException";
+const char * kPathNotFoundException = "org.apache.hadoop.fs.InvalidPathException";
+const char * kPathNotFoundException2 = "java.io.FileNotFoundException";
+const char * kFileAlreadyExistsException = "org.apache.hadoop.fs.FileAlreadyExistsException";
const char * kPathIsNotEmptyDirectoryException = "org.apache.hadoop.fs.PathIsNotEmptyDirectoryException";
-Status::Status(int code, const char *msg1) : code_(code) {
+
+const static std::map<std::string, int> kKnownServerExceptionClasses = {
+ {kStatusAccessControlException, Status::kAccessControlException},
+ {kPathIsNotDirectoryException, Status::kNotADirectory},
+ {kSnapshotException, Status::kSnapshotProtocolException},
+ {kStatusStandbyException, Status::kStandbyException},
+ {kStatusSaslException, Status::kAuthenticationFailed},
+ {kPathNotFoundException, Status::kPathNotFound},
+ {kPathNotFoundException2, Status::kPathNotFound},
+ {kFileAlreadyExistsException, Status::kFileAlreadyExists},
+ {kPathIsNotEmptyDirectoryException, Status::kPathIsNotEmptyDirectory}
+ };
+
+
+Status::Status(int code, const char *msg1)
+ : code_(code) {
if(msg1) {
msg_ = msg1;
}
}
-Status::Status(int code, const char *msg1, const char *msg2) : code_(code) {
- std::stringstream ss;
- if(msg1) {
- ss << msg1;
- if(msg2) {
- ss << ":" << msg2;
- }
+Status::Status(int code, const char *exception_class_name, const char *exception_details)
+ : code_(code) {
+ // If we can assure this never gets nullptr args this can be
+ // in the initializer list.
+ if(exception_class_name)
+ exception_class_ = exception_class_name;
+ if(exception_details)
+ msg_ = exception_details;
+
+ std::map<std::string, int>::const_iterator it = kKnownServerExceptionClasses.find(exception_class_);
+ if(it != kKnownServerExceptionClasses.end()) {
+ code_ = it->second;
}
- msg_ = ss.str();
}
@@ -72,6 +94,7 @@ Status Status::Unimplemented() {
}
Status Status::Exception(const char *exception_class_name, const char *error_message) {
+ // Server side exception but can be represented by std::errc codes
if (exception_class_name && (strcmp(exception_class_name, kStatusAccessControlException) == 0) )
return Status(kPermissionDenied, error_message);
else if (exception_class_name && (strcmp(exception_class_name, kStatusSaslException) == 0))
@@ -81,13 +104,13 @@ Status Status::Exception(const char *exception_class_name, const char *error_mes
else if (exception_class_name && (strcmp(exception_class_name, kPathNotFoundException2) == 0))
return Status(kPathNotFound, error_message);
else if (exception_class_name && (strcmp(exception_class_name, kPathIsNotDirectoryException) == 0))
- return Status(kNotADirectory, error_message);
+ return Status(kNotADirectory, error_message);
else if (exception_class_name && (strcmp(exception_class_name, kSnapshotException) == 0))
- return Status(kInvalidArgument, error_message);
+ return Status(kInvalidArgument, error_message);
else if (exception_class_name && (strcmp(exception_class_name, kFileAlreadyExistsException) == 0))
- return Status(kFileAlreadyExists, error_message);
+ return Status(kFileAlreadyExists, error_message);
else if (exception_class_name && (strcmp(exception_class_name, kPathIsNotEmptyDirectoryException) == 0))
- return Status(kPathIsNotEmptyDirectory, error_message);
+ return Status(kPathIsNotEmptyDirectory, error_message);
else
return Status(kException, exception_class_name, error_message);
}
@@ -101,15 +124,19 @@ Status Status::AuthenticationFailed() {
}
Status Status::Canceled() {
- return Status(kOperationCanceled,"Operation canceled");
+ return Status(kOperationCanceled, "Operation canceled");
}
-
std::string Status::ToString() const {
if (code_ == kOk) {
return "OK";
}
- return msg_;
+ std::stringstream ss;
+ if(!exception_class_.empty()) {
+ ss << exception_class_ << ":";
+ }
+ ss << msg_;
+ return ss.str();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.cc
index 616cf3f..a54be8a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.cc
@@ -133,11 +133,11 @@ std::vector<std::string> split(const std::string input, char separator)
std::string copy_range(const UriTextRangeA *r) {
- const int size = r->afterLast - r->first;
- if (size) {
+ const int size = r->afterLast - r->first;
+ if (size) {
return std::string(r->first, size);
- }
- return "";
+ }
+ return "";
}
bool parse_int(const UriTextRangeA *r, optional<uint16_t> * result) {
@@ -161,14 +161,14 @@ bool parse_int(const UriTextRangeA *r, optional<uint16_t> * result) {
std::vector<std::string> copy_path(const UriPathSegmentA *ps) {
std::vector<std::string> result;
- if (nullptr == ps)
+ if (nullptr == ps)
return result;
- for (; ps != 0; ps = ps->next) {
- result.push_back(copy_range(&ps->text));
- }
+ for (; ps != 0; ps = ps->next) {
+ result.push_back(copy_range(&ps->text));
+ }
- return result;
+ return result;
}
void parse_user_info(const UriTextRangeA *r, std::string * user, std::string * pass) {
@@ -204,7 +204,7 @@ std::vector<std::pair<std::string, std::string > > parse_query(const char *first
uriFreeQueryListA(query);
}
- return result;
+ return result;
}
@@ -368,4 +368,32 @@ void URI::remove_queries(const std::string &q_name, bool encoded_input)
}
}
+std::string URI::GetDebugString() const {
+ std::stringstream ss;
+ ss << std::endl;
+ ss << "\t" << "uri.str() = \"" << str() << "\"" << std::endl;
+ ss << "\t" << "uri.get_scheme() = \"" << get_scheme() << "\"" << std::endl;
+ ss << "\t" << "uri.get_host() = \"" << get_host() << "\"" << std::endl;
+
+ if(!port)
+ ss << "\t" << "uri.get_port() = unset optional<uint16_t>" << std::endl;
+ else
+ ss << "\t" << "uri.get_port() = \"" << port.value() << "\"" << std::endl;
+
+ ss << "\t" << "uri.get_path() = \"" << get_path() << "\"" << std::endl;
+ ss << "\t" << "uri.get_fragment() = \"" << get_fragment() << "\"" << std::endl;
+
+
+ std::vector<std::pair<std::string, std::string> > elems = get_query_elements();
+
+ if(elems.size() > 0)
+ ss << "\t" << "Query elements:" << std::endl;
+
+ for(auto qry = elems.begin(); qry != elems.end(); qry++) {
+ ss << "\t\t" << qry->first << " -> " << qry->second << std::endl;
+ }
+
+ return ss.str();
+}
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.h
index e6fbd78..87f6919 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.h
@@ -118,11 +118,13 @@ public:
{ fragment = to_encoded(encoded_input,f); }
std::string str(bool encoded_output=true) const;
+
+ // Get a string with each URI field printed on a seperate line
+ std::string GetDebugString() const;
};
inline std::ostream& operator<<(std::ostream &out, const URI &uri)
{ return out << uri.str(); }
}
-
-#endif
\ No newline at end of file
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
index f3c32c2..ae55e0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
@@ -18,6 +18,8 @@
#include "filesystem.h"
+#include "common/namenode_info.h"
+
#include <functional>
#include <limits>
#include <future>
@@ -126,9 +128,35 @@ void FileSystemImpl::Connect(const std::string &server,
handler (Status::Error("Null IoService"), this);
}
- cluster_name_ = server + ":" + service;
+ // DNS lookup here for namenode(s)
+ std::vector<ResolvedNamenodeInfo> resolved_namenodes;
+
+ auto name_service = options_.services.find(server);
+ if(name_service != options_.services.end()) {
+ cluster_name_ = name_service->first;
+ resolved_namenodes = BulkResolve(&io_service_->io_service(), name_service->second);
+ } else {
+ cluster_name_ = server + ":" + service;
+
+ // tmp namenode info just to get this in the right format for BulkResolve
+ NamenodeInfo tmp_info;
+ optional<URI> uri = URI::parse_from_string("hdfs://" + cluster_name_);
+ if(!uri) {
+ LOG_ERROR(kFileSystem, << "Unable to use URI for cluster " << cluster_name_);
+ handler(Status::Error(("Invalid namenode " + cluster_name_ + " in config").c_str()), this);
+ }
+ tmp_info.uri = uri.value();
+
+ resolved_namenodes = BulkResolve(&io_service_->io_service(), {tmp_info});
+ }
+
+ for(unsigned int i=0;i<resolved_namenodes.size();i++) {
+ LOG_DEBUG(kFileSystem, << "Resolved Namenode");
+ LOG_DEBUG(kFileSystem, << resolved_namenodes[i].str());
+ }
+
- nn_.Connect(cluster_name_, server, service, [this, handler](const Status & s) {
+ nn_.Connect(cluster_name_, /*server, service*/ resolved_namenodes, [this, handler](const Status & s) {
handler(s, this);
});
}
@@ -216,6 +244,13 @@ void FileSystemImpl::Open(
<< path << ") called");
nn_.GetBlockLocations(path, [this, path, handler](const Status &stat, std::shared_ptr<const struct FileInfo> file_info) {
+ if(!stat.ok()) {
+ LOG_INFO(kFileSystem, << "FileSystemImpl::Open failed to get block locations. status=" << stat.ToString());
+ if(stat.get_server_exception_type() == Status::kStandbyException) {
+ LOG_INFO(kFileSystem, << "Operation not allowed on standby datanode");
+ }
+ }
+
handler(stat, stat.ok() ? new FileHandleImpl(cluster_name_, path, &io_service_->io_service(), client_name_, file_info, bad_node_tracker_, event_handlers_)
: nullptr);
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
index ee566ea..8b4c126 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
@@ -27,6 +27,7 @@
#include <tuple>
#include <iostream>
#include <pwd.h>
+#include <utility>
#define FMT_THIS_ADDR "this=" << (void*)this
@@ -51,20 +52,9 @@ Status NameNodeOperations::CheckValidPermissionMask(short permissions) {
}
void NameNodeOperations::Connect(const std::string &cluster_name,
- const std::string &server,
- const std::string &service,
- std::function<void(const Status &)> &&handler) {
- using namespace asio_continuation;
- typedef std::vector<tcp::endpoint> State;
- auto m = Pipeline<State>::Create();
- m->Push(Resolve(io_service_, server, service,
- std::back_inserter(m->state())))
- .Push(Bind([this, m, cluster_name](const Continuation::Next &next) {
- engine_.Connect(cluster_name, m->state(), next);
- }));
- m->Run([this, handler](const Status &status, const State &) {
- handler(status);
- });
+ const std::vector<ResolvedNamenodeInfo> &servers,
+ std::function<void(const Status &)> &&handler) {
+ engine_.Connect(cluster_name, servers, handler);
}
void NameNodeOperations::GetBlockLocations(const std::string & path,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
index 4f4d63e..9651570 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
@@ -21,6 +21,7 @@
#include "rpc/rpc_engine.h"
#include "hdfspp/statinfo.h"
#include "hdfspp/fsinfo.h"
+#include "common/namenode_info.h"
#include "ClientNamenodeProtocol.pb.h"
#include "ClientNamenodeProtocol.hrpc.inl"
@@ -45,13 +46,12 @@ public:
const char *protocol_name, int protocol_version) :
io_service_(io_service),
engine_(io_service, options, client_name, user_name, protocol_name, protocol_version),
- namenode_(& engine_) {}
+ namenode_(& engine_), options_(options) {}
static Status CheckValidPermissionMask(short permissions);
void Connect(const std::string &cluster_name,
- const std::string &server,
- const std::string &service,
+ const std::vector<ResolvedNamenodeInfo> &servers,
std::function<void(const Status &)> &&handler);
void GetBlockLocations(const std::string & path,
@@ -104,6 +104,7 @@ private:
::asio::io_service * io_service_;
RpcEngine engine_;
ClientNamenodeProtocol namenode_;
+ const Options options_;
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
index be6d7bd..03f83f4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
@@ -118,7 +118,8 @@ Request::Request(LockFreeRpcEngine *engine, const std::string &method_name, int
call_id_(call_id),
timer_(engine->io_service()),
handler_(std::move(handler)),
- retry_count_(engine->retry_policy() ? 0 : kNoRetry) {
+ retry_count_(engine->retry_policy() ? 0 : kNoRetry),
+ failover_count_(0) {
ConstructPayload(&payload_, &request);
}
@@ -129,7 +130,8 @@ Request::Request(LockFreeRpcEngine *engine, const std::string &method_name, int
call_id_(call_id),
timer_(engine->io_service()),
handler_(std::move(handler)),
- retry_count_(engine->retry_policy() ? 0 : kNoRetry) {
+ retry_count_(engine->retry_policy() ? 0 : kNoRetry),
+ failover_count_(0) {
ConstructPayload(&payload_, request);
}
@@ -138,10 +140,13 @@ Request::Request(LockFreeRpcEngine *engine, Handler &&handler)
call_id_(-1),
timer_(engine->io_service()),
handler_(std::move(handler)),
- retry_count_(engine->retry_policy() ? 0 : kNoRetry) {
+ retry_count_(engine->retry_policy() ? 0 : kNoRetry),
+ failover_count_(0) {
}
void Request::GetPacket(std::string *res) const {
+ LOG_TRACE(kRPC, << "Request::GetPacket called");
+
if (payload_.empty())
return;
@@ -159,9 +164,27 @@ void Request::GetPacket(std::string *res) const {
void Request::OnResponseArrived(pbio::CodedInputStream *is,
const Status &status) {
+ LOG_TRACE(kRPC, << "Request::OnResponseArrived called");
handler_(is, status);
}
+std::string Request::GetDebugString() const {
+ // Basic description of this object, aimed at debugging
+ std::stringstream ss;
+ ss << "\nRequest Object:\n";
+ ss << "\tMethod name = \"" << method_name_ << "\"\n";
+ ss << "\tCall id = " << call_id_ << "\n";
+ ss << "\tRetry Count = " << retry_count_ << "\n";
+ ss << "\tFailover count = " << failover_count_ << "\n";
+ return ss.str();
+}
+
+int Request::IncrementFailoverCount() {
+ // reset retry count when failing over
+ retry_count_ = 0;
+ return failover_count_++;
+}
+
RpcConnection::RpcConnection(LockFreeRpcEngine *engine)
: engine_(engine),
connected_(kNotYetConnected) {}
@@ -258,7 +281,7 @@ void RpcConnection::AsyncFlushPendingRequests() {
});
}
-void RpcConnection::HandleRpcResponse(std::shared_ptr<Response> response) {
+Status RpcConnection::HandleRpcResponse(std::shared_ptr<Response> response) {
assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
response->ar.reset(new pbio::ArrayInputStream(&response->data_[0], response->data_.size()));
@@ -270,7 +293,7 @@ void RpcConnection::HandleRpcResponse(std::shared_ptr<Response> response) {
auto req = RemoveFromRunningQueue(h.callid());
if (!req) {
LOG_WARN(kRPC, << "RPC response with Unknown call id " << h.callid());
- return;
+ return Status::Error("Rpc response with unknown call id");
}
Status status;
@@ -288,9 +311,22 @@ void RpcConnection::HandleRpcResponse(std::shared_ptr<Response> response) {
Status::Exception(h.exceptionclassname().c_str(), h.errormsg().c_str());
}
+ if(status.get_server_exception_type() == Status::kStandbyException) {
+ LOG_WARN(kRPC, << "Tried to connect to standby. status = " << status.ToString());
+
+ // We got the request back, but it needs to be resent to the other NN
+ std::vector<std::shared_ptr<Request>> reqs_to_redirect = {req};
+ PrependRequests_locked(reqs_to_redirect);
+
+ CommsError(status);
+ return status;
+ }
+
io_service().post([req, response, status]() {
req->OnResponseArrived(response->in.get(), status); // Never call back while holding a lock
});
+
+ return Status::OK();
}
void RpcConnection::HandleRpcTimeout(std::shared_ptr<Request> req,
@@ -437,6 +473,15 @@ void RpcConnection::PreEnqueueRequests(
// Don't start sending yet; will flush when connected
}
+// Only call when already holding conn state lock
+void RpcConnection::PrependRequests_locked( std::vector<std::shared_ptr<Request>> requests) {
+ LOG_DEBUG(kRPC, << "RpcConnection::PrependRequests called");
+
+ pending_requests_.insert(pending_requests_.begin(), requests.begin(),
+ requests.end());
+ // Don't start sending yet; will flush when connected
+}
+
void RpcConnection::SetEventHandlers(std::shared_ptr<LibhdfsEvents> event_handlers) {
std::lock_guard<std::mutex> state_lock(connection_state_lock_);
event_handlers_ = event_handlers;
@@ -452,6 +497,7 @@ void RpcConnection::SetClusterName(std::string cluster_name) {
void RpcConnection::CommsError(const Status &status) {
assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
+ LOG_DEBUG(kRPC, << "RpcConnection::CommsError called");
Disconnect();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
index 330f9b1..869be40 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
@@ -112,6 +112,8 @@ void RpcConnectionImpl<NextLayer>::Connect(
template <class NextLayer>
void RpcConnectionImpl<NextLayer>::ConnectAndFlush(
const std::vector<::asio::ip::tcp::endpoint> &server) {
+
+ LOG_INFO(kRPC, << "ConnectAndFlush called");
std::lock_guard<std::mutex> state_lock(connection_state_lock_);
if (server.empty()) {
@@ -408,7 +410,16 @@ void RpcConnectionImpl<NextLayer>::OnRecvCompleted(const ::asio::error_code &ori
OnRecvCompleted(ec, size);
});
} else if (current_response_state_->state_ == Response::kParseResponse) {
- HandleRpcResponse(current_response_state_);
+ // Check return status from the RPC response. We may have received a msg
+ // indicating a server side error.
+
+ Status stat = HandleRpcResponse(current_response_state_);
+
+ if(stat.get_server_exception_type() == Status::kStandbyException) {
+ // May need to bail out, connect to new NN, and restart loop
+ LOG_INFO(kRPC, << "Communicating with standby NN, attempting to reconnect");
+ }
+
current_response_state_ = nullptr;
StartReading();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
index a8438b1..be69d95 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
@@ -19,15 +19,126 @@
#include "rpc_connection.h"
#include "common/util.h"
#include "common/logging.h"
+#include "common/namenode_info.h"
#include "optional.hpp"
#include <future>
+#include <algorithm>
namespace hdfs {
template <class T>
using optional = std::experimental::optional<T>;
+HANamenodeTracker::HANamenodeTracker(const std::vector<ResolvedNamenodeInfo> &servers,
+ ::asio::io_service *ioservice,
+ std::shared_ptr<LibhdfsEvents> event_handlers)
+ : enabled_(false), resolved_(false),
+ ioservice_(ioservice), event_handlers_(event_handlers)
+{
+ LOG_TRACE(kRPC, << "HANamenodeTracker got the following nodes");
+ for(unsigned int i=0;i<servers.size();i++)
+ LOG_TRACE(kRPC, << servers[i].str());
+
+ if(servers.size() >= 2) {
+ LOG_TRACE(kRPC, << "Creating HA namenode tracker");
+ if(servers.size() > 2) {
+ LOG_WARN(kRPC, << "Nameservice declares more than two nodes. Some won't be used.");
+ }
+
+ active_info_ = servers[0];
+ standby_info_ = servers[1];
+ LOG_INFO(kRPC, << "Active namenode url = " << active_info_.uri.str());
+ LOG_INFO(kRPC, << "Standby namenode url = " << standby_info_.uri.str());
+
+ enabled_ = true;
+ if(!active_info_.endpoints.empty() || !standby_info_.endpoints.empty()) {
+ resolved_ = true;
+ }
+ }
+}
+
+
+HANamenodeTracker::~HANamenodeTracker() { }
+
+
+static std::string format_endpoints(const std::vector<::asio::ip::tcp::endpoint> &pts) {
+ std::stringstream ss;
+ for(unsigned int i=0; i<pts.size(); i++)
+ if(i == pts.size() - 1)
+ ss << pts[i];
+ else
+ ss << pts[i] << ", ";
+ return ss.str();
+}
+
+// Pass in endpoint from current connection, this will do a reverse lookup
+// and return the info for the standby node. It will also swap its state internally.
+ResolvedNamenodeInfo HANamenodeTracker::GetFailoverAndUpdate(::asio::ip::tcp::endpoint current_endpoint) {
+ LOG_TRACE(kRPC, << "Swapping from endpoint " << current_endpoint);
+ mutex_guard swap_lock(swap_lock_);
+
+ ResolvedNamenodeInfo failover_node;
+
+ // Connected to standby, switch standby to active
+ if(IsCurrentActive_locked(current_endpoint)) {
+ std::swap(active_info_, standby_info_);
+ if(event_handlers_)
+ event_handlers_->call(FS_NN_FAILOVER_EVENT, active_info_.nameservice.c_str(),
+ reinterpret_cast<int64_t>(active_info_.uri.str().c_str()));
+ failover_node = active_info_;
+ } else if(IsCurrentStandby_locked(current_endpoint)) {
+ // Connected to standby
+ if(event_handlers_)
+ event_handlers_->call(FS_NN_FAILOVER_EVENT, active_info_.nameservice.c_str(),
+ reinterpret_cast<int64_t>(active_info_.uri.str().c_str()));
+ failover_node = active_info_;
+ } else {
+ // Invalid state, throw for testing
+ std::string ep1 = format_endpoints(active_info_.endpoints);
+ std::string ep2 = format_endpoints(standby_info_.endpoints);
+
+ std::stringstream msg;
+ msg << "Looked for " << current_endpoint << " in\n";
+ msg << ep1 << " and\n";
+ msg << ep2 << std::endl;
+
+ LOG_ERROR(kRPC, << "Unable to find RPC connection in config " << msg.str() << ". Bailing out.");
+ throw std::runtime_error(msg.str());
+ }
+
+ if(failover_node.endpoints.empty()) {
+ LOG_WARN(kRPC, << "No endpoints for node " << failover_node.uri.str() << " attempting to resolve again");
+ if(!ResolveInPlace(ioservice_, failover_node)) {
+ LOG_ERROR(kRPC, << "Fallback endpoint resolution for node " << failover_node.uri.str()
+ << "failed. Please make sure your configuration is up to date.");
+ }
+ }
+ return failover_node;
+}
+
+bool HANamenodeTracker::IsCurrentActive_locked(const ::asio::ip::tcp::endpoint &ep) const {
+ for(unsigned int i=0;i<active_info_.endpoints.size();i++) {
+ if(ep.address() == active_info_.endpoints[i].address()) {
+ if(ep.port() != active_info_.endpoints[i].port())
+ LOG_WARN(kRPC, << "Port mismatch: " << ep << " vs " << active_info_.endpoints[i] << " trying anyway..");
+ return true;
+ }
+ }
+ return false;
+}
+
+bool HANamenodeTracker::IsCurrentStandby_locked(const ::asio::ip::tcp::endpoint &ep) const {
+ for(unsigned int i=0;i<standby_info_.endpoints.size();i++) {
+ if(ep.address() == standby_info_.endpoints[i].address()) {
+ if(ep.port() != standby_info_.endpoints[i].port())
+ LOG_WARN(kRPC, << "Port mismatch: " << ep << " vs " << standby_info_.endpoints[i] << " trying anyway..");
+ return true;
+ }
+ }
+ return false;
+}
+
RpcEngine::RpcEngine(::asio::io_service *io_service, const Options &options,
const std::string &client_name, const std::string &user_name,
const char *protocol_name, int protocol_version)
@@ -36,27 +147,35 @@ RpcEngine::RpcEngine(::asio::io_service *io_service, const Options &options,
client_name_(client_name),
protocol_name_(protocol_name),
protocol_version_(protocol_version),
- retry_policy_(std::move(MakeRetryPolicy(options))),
call_id_(0),
retry_timer(*io_service),
- event_handlers_(std::make_shared<LibhdfsEvents>()) {
-
- auth_info_.setUser(user_name);
- if (options.authentication == Options::kKerberos) {
- auth_info_.setMethod(AuthInfo::kKerberos);
- }
+ event_handlers_(std::make_shared<LibhdfsEvents>())
+{
+ LOG_DEBUG(kRPC, << "RpcEngine::RpcEngine called");
- LOG_DEBUG(kRPC, << "RpcEngine::RpcEngine called");
+ auth_info_.setUser(user_name);
+ if (options.authentication == Options::kKerberos) {
+ auth_info_.setMethod(AuthInfo::kKerberos);
+ }
}
void RpcEngine::Connect(const std::string &cluster_name,
- const std::vector<::asio::ip::tcp::endpoint> &server,
+ const std::vector<ResolvedNamenodeInfo> servers,
RpcCallback &handler) {
std::lock_guard<std::mutex> state_lock(engine_state_lock_);
LOG_DEBUG(kRPC, << "RpcEngine::Connect called");
- last_endpoints_ = server;
+ last_endpoints_ = servers[0].endpoints;
cluster_name_ = cluster_name;
+ LOG_TRACE(kRPC, << "Got cluster name \"" << cluster_name << "\" in RpcEngine::Connect")
+
+ ha_persisted_info_.reset(new HANamenodeTracker(servers, io_service_, event_handlers_));
+ if(!ha_persisted_info_->is_enabled()) {
+ ha_persisted_info_.reset();
+ }
+
+ // Construct retry policy after we determine if config is HA
+ retry_policy_ = std::move(MakeRetryPolicy(options_));
conn_ = InitializeConnection();
conn_->Connect(last_endpoints_, auth_info_, handler);
@@ -72,8 +191,16 @@ void RpcEngine::Shutdown() {
std::unique_ptr<const RetryPolicy> RpcEngine::MakeRetryPolicy(const Options &options) {
LOG_DEBUG(kRPC, << "RpcEngine::MakeRetryPolicy called");
- if (options.max_rpc_retries > 0) {
- return std::unique_ptr<RetryPolicy>(new FixedDelayRetryPolicy(options.rpc_retry_delay_ms, options.max_rpc_retries));
+
+ if(ha_persisted_info_) {
+ LOG_INFO(kRPC, << "Cluster is HA configued so policy will default to HA until a knob is implemented");
+ return std::unique_ptr<RetryPolicy>(new FixedDelayWithFailover(options.rpc_retry_delay_ms,
+ options.max_rpc_retries,
+ options.failover_max_retries,
+ options.failover_connection_max_retries));
+ } else if (options.max_rpc_retries > 0) {
+ return std::unique_ptr<RetryPolicy>(new FixedDelayRetryPolicy(options.rpc_retry_delay_ms,
+ options.max_rpc_retries));
} else {
return nullptr;
}
@@ -81,6 +208,15 @@ std::unique_ptr<const RetryPolicy> RpcEngine::MakeRetryPolicy(const Options &opt
void RpcEngine::TEST_SetRpcConnection(std::shared_ptr<RpcConnection> conn) {
conn_ = conn;
+ retry_policy_ = std::move(MakeRetryPolicy(options_));
+}
+
+void RpcEngine::TEST_SetRetryPolicy(std::unique_ptr<const RetryPolicy> policy) {
+ retry_policy_ = std::move(policy);
+}
+
+std::unique_ptr<const RetryPolicy> RpcEngine::TEST_GenerateRetryPolicyUsingOptions() {
+ return MakeRetryPolicy(options_);
}
void RpcEngine::AsyncRpc(
@@ -131,7 +267,7 @@ void RpcEngine::AsyncRpcCommsError(
const Status &status,
std::shared_ptr<RpcConnection> failedConnection,
std::vector<std::shared_ptr<Request>> pendingRequests) {
- LOG_ERROR(kRPC, << "RpcEngine::AsyncRpcCommsError called; conn=" << failedConnection.get() << " reqs=" << pendingRequests.size());
+ LOG_ERROR(kRPC, << "RpcEngine::AsyncRpcCommsError called; status=\"" << status.ToString() << "\" conn=" << failedConnection.get() << " reqs=" << pendingRequests.size());
io_service().post([this, status, failedConnection, pendingRequests]() {
RpcCommsError(status, failedConnection, pendingRequests);
@@ -142,44 +278,52 @@ void RpcEngine::RpcCommsError(
const Status &status,
std::shared_ptr<RpcConnection> failedConnection,
std::vector<std::shared_ptr<Request>> pendingRequests) {
- (void)status;
-
- LOG_ERROR(kRPC, << "RpcEngine::RpcCommsError called; conn=" << failedConnection.get() << " reqs=" << pendingRequests.size());
+ LOG_WARN(kRPC, << "RpcEngine::RpcCommsError called; status=\"" << status.ToString() << "\" conn=" << failedConnection.get() << " reqs=" << pendingRequests.size());
std::lock_guard<std::mutex> state_lock(engine_state_lock_);
// If the failed connection is the current one, shut it down
// It will be reconnected when there is work to do
if (failedConnection == conn_) {
- conn_.reset();
+ LOG_INFO(kRPC, << "Disconnecting from failed RpcConnection");
+ conn_.reset();
}
- auto head_action = optional<RetryAction>();
+ optional<RetryAction> head_action = optional<RetryAction>();
- // Filter out anything with too many retries already
- for (auto it = pendingRequests.begin(); it < pendingRequests.end();) {
- auto req = *it;
+ //We are talking to the Standby NN, let's talk to the active one instead.
+ if(ha_persisted_info_ && status.get_server_exception_type() == Status::kStandbyException) {
+ LOG_INFO(kRPC, << "Received StandbyException. Failing over.");
+ head_action = RetryAction::failover(std::max(0,options_.rpc_retry_delay_ms));
+ } else {
+ // Filter out anything with too many retries already
+ for (auto it = pendingRequests.begin(); it < pendingRequests.end();) {
+ auto req = *it;
- RetryAction retry = RetryAction::fail(""); // Default to fail
- if (retry_policy()) {
- retry = retry_policy()->ShouldRetry(status, req->IncrementRetryCount(), 0, true);
- }
+ LOG_DEBUG(kRPC, << req->GetDebugString());
- if (retry.action == RetryAction::FAIL) {
- // If we've exceeded the maximum retry, take the latest error and pass it
- // on. There might be a good argument for caching the first error
- // rather than the last one, that gets messy
+ RetryAction retry = RetryAction::fail(""); // Default to fail
- io_service().post([req, status]() {
- req->OnResponseArrived(nullptr, status); // Never call back while holding a lock
- });
- it = pendingRequests.erase(it);
- } else {
- if (!head_action) {
- head_action = retry;
+ if (retry_policy()) {
+ retry = retry_policy()->ShouldRetry(status, req->IncrementRetryCount(), req->get_failover_count(), true);
}
- ++it;
+ if (retry.action == RetryAction::FAIL) {
+ // If we've exceeded the maximum retry, take the latest error and pass it
+ // on. There might be a good argument for caching the first error
+ // rather than the last one, that gets messy
+
+ io_service().post([req, status]() {
+ req->OnResponseArrived(nullptr, status); // Never call back while holding a lock
+ });
+ it = pendingRequests.erase(it);
+ } else {
+ if (!head_action) {
+ head_action = retry;
+ }
+
+ ++it;
+ }
}
}
@@ -189,8 +333,31 @@ void RpcEngine::RpcCommsError(
head_action && head_action->action != RetryAction::FAIL;
if (haveRequests) {
+ LOG_TRACE(kRPC, << "Have " << pendingRequests.size() << " requests to resend");
bool needNewConnection = !conn_;
if (needNewConnection) {
+ LOG_DEBUG(kRPC, << "Creating a new NN conection");
+
+
+ // If HA is enabled and we have valid HA info then fail over to the standby (hopefully now active)
+ if(head_action->action == RetryAction::FAILOVER_AND_RETRY && ha_persisted_info_) {
+
+ for(unsigned int i=0; i<pendingRequests.size();i++)
+ pendingRequests[i]->IncrementFailoverCount();
+
+ ResolvedNamenodeInfo new_active_nn_info =
+ ha_persisted_info_->GetFailoverAndUpdate(last_endpoints_[0]/*reverse lookup*/);
+
+ LOG_INFO(kRPC, << "Going to try connecting to alternate Datanode: " << new_active_nn_info.uri.str());
+
+ if(ha_persisted_info_->is_resolved()) {
+ last_endpoints_ = new_active_nn_info.endpoints;
+ } else {
+ LOG_WARN(kRPC, << "It looks HA is turned on, but unable to fail over. has info="
+ << ha_persisted_info_->is_enabled() << " resolved=" << ha_persisted_info_->is_resolved());
+ }
+ }
+
conn_ = InitializeConnection();
conn_->PreEnqueueRequests(pendingRequests);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
index 5de7d53..47618a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
@@ -24,7 +24,11 @@
#include "common/auth_info.h"
#include "common/retry_policy.h"
#include "common/libhdfs_events_impl.h"
+#include "common/util.h"
+#include "common/continuation/asio.h"
+#include "common/logging.h"
#include "common/new_delete.h"
+#include "common/namenode_info.h"
#include <google/protobuf/message_lite.h>
#include <google/protobuf/io/coded_stream.h>
@@ -38,6 +42,7 @@
#include <unordered_map>
#include <vector>
#include <mutex>
+#include <future>
namespace hdfs {
@@ -84,10 +89,15 @@ class Request {
std::string method_name() const { return method_name_; }
::asio::deadline_timer &timer() { return timer_; }
int IncrementRetryCount() { return retry_count_++; }
+ int IncrementFailoverCount();
void GetPacket(std::string *res) const;
void OnResponseArrived(::google::protobuf::io::CodedInputStream *is,
const Status &status);
+ int get_failover_count() {return failover_count_;}
+
+ std::string GetDebugString() const;
+
private:
LockFreeRpcEngine *const engine_;
const std::string method_name_;
@@ -98,6 +108,7 @@ class Request {
const Handler handler_;
int retry_count_;
+ int failover_count_;
};
/*
@@ -137,6 +148,9 @@ class RpcConnection : public std::enable_shared_from_this<RpcConnection> {
// on connect
void PreEnqueueRequests(std::vector<std::shared_ptr<Request>> requests);
+ // Put requests at the front of the current request queue
+ void PrependRequests_locked(std::vector<std::shared_ptr<Request>> requests);
+
void SetEventHandlers(std::shared_ptr<LibhdfsEvents> event_handlers);
void SetClusterName(std::string cluster_name);
@@ -190,7 +204,7 @@ class RpcConnection : public std::enable_shared_from_this<RpcConnection> {
static std::string SerializeRpcRequest(
const std::string &method_name,
const ::google::protobuf::MessageLite *req);
- void HandleRpcResponse(std::shared_ptr<Response> response);
+ Status HandleRpcResponse(std::shared_ptr<Response> response);
void HandleRpcTimeout(std::shared_ptr<Request> req,
const ::asio::error_code &ec);
void CommsError(const Status &status);
@@ -261,6 +275,55 @@ public:
virtual const Options &options() const = 0;
};
+
+/*
+ * Tracker gives the RpcEngine a quick way to use an endpoint that just
+ * failed in order to lookup a set of endpoints for a failover node.
+ *
+ * Note: For now this only deals with 2 NameNodes, but that's the default
+ * anyway.
+ */
+class HANamenodeTracker {
+ public:
+ HANamenodeTracker(const std::vector<ResolvedNamenodeInfo> &servers,
+ ::asio::io_service *ioservice,
+ std::shared_ptr<LibhdfsEvents> event_handlers_);
+
+ virtual ~HANamenodeTracker();
+
+ bool is_enabled() const { return enabled_; }
+ bool is_resolved() const { return resolved_; }
+
+ // Get node opposite of the current one if possible (swaps active/standby)
+ // Note: This will always mutate internal state. Use IsCurrentActive/Standby to
+ // get info without changing state
+ ResolvedNamenodeInfo GetFailoverAndUpdate(::asio::ip::tcp::endpoint current_endpoint);
+
+ bool IsCurrentActive_locked(const ::asio::ip::tcp::endpoint &ep) const;
+ bool IsCurrentStandby_locked(const ::asio::ip::tcp::endpoint &ep) const;
+
+ private:
+ // If HA should be enabled, according to our options and runtime info like # nodes provided
+ bool enabled_;
+ // If we were able to resolve at least 1 HA namenode
+ bool resolved_;
+
+ // Keep service in case a second round of DNS lookup is required
+ ::asio::io_service *ioservice_;
+
+ // Event handlers, for now this is the simplest place to catch all failover events
+ // and push info out to client application. Possibly move into RPCEngine.
+ std::shared_ptr<LibhdfsEvents> event_handlers_;
+
+ // Only support 1 active and 1 standby for now.
+ ResolvedNamenodeInfo active_info_;
+ ResolvedNamenodeInfo standby_info_;
+
+ // Aquire when switching from active-standby
+ std::mutex swap_lock_;
+};
+
+
/*
* An engine for reliable communication with a NameNode. Handles connection,
* retry, and (someday) failover of the requested messages.
@@ -285,7 +348,7 @@ class RpcEngine : public LockFreeRpcEngine {
const char *protocol_name, int protocol_version);
void Connect(const std::string & cluster_name,
- const std::vector<::asio::ip::tcp::endpoint> &server,
+ const std::vector<ResolvedNamenodeInfo> servers,
RpcCallback &handler);
void AsyncRpc(const std::string &method_name,
@@ -297,7 +360,6 @@ class RpcEngine : public LockFreeRpcEngine {
const ::google::protobuf::MessageLite *req,
const std::shared_ptr<::google::protobuf::MessageLite> &resp);
- void Start();
void Shutdown();
/* Enqueues a CommsError without acquiring a lock*/
@@ -313,6 +375,8 @@ class RpcEngine : public LockFreeRpcEngine {
int NextCallId() override { return ++call_id_; }
void TEST_SetRpcConnection(std::shared_ptr<RpcConnection> conn);
+ void TEST_SetRetryPolicy(std::unique_ptr<const RetryPolicy> policy);
+ std::unique_ptr<const RetryPolicy> TEST_GenerateRetryPolicyUsingOptions();
const std::string &client_name() const override { return client_name_; }
const std::string &user_name() const override { return auth_info_.getUser(); }
@@ -338,7 +402,7 @@ private:
const std::string client_name_;
const std::string protocol_name_;
const int protocol_version_;
- const std::unique_ptr<const RetryPolicy> retry_policy_; //null --> no retry
+ std::unique_ptr<const RetryPolicy> retry_policy_; //null --> no retry
AuthInfo auth_info_;
std::string cluster_name_;
std::atomic_int call_id_;
@@ -348,6 +412,9 @@ private:
std::mutex engine_state_lock_;
+ // Keep endpoint info for all HA connections, a non-null ptr indicates
+ // that HA info was found in the configuation.
+ std::unique_ptr<HANamenodeTracker> ha_persisted_info_;
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb0dad5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
index defe95d..3e8c93f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
@@ -20,6 +20,7 @@
#include "test.pb.h"
#include "RpcHeader.pb.h"
#include "rpc/rpc_connection.h"
+#include "common/namenode_info.h"
#include <google/protobuf/io/coded_stream.h>
@@ -43,10 +44,10 @@ namespace pbio = ::google::protobuf::io;
namespace hdfs {
-std::vector<asio::ip::basic_endpoint<asio::ip::tcp>> make_endpoint() {
- std::vector<asio::ip::basic_endpoint<asio::ip::tcp>> result;
- result.push_back(asio::ip::basic_endpoint<asio::ip::tcp>());
- return result;
+std::vector<ResolvedNamenodeInfo> make_endpoint() {
+ ResolvedNamenodeInfo result;
+ result.endpoints.push_back(asio::ip::basic_endpoint<asio::ip::tcp>());
+ return std::vector<ResolvedNamenodeInfo>({result});
}
class MockRPCConnection : public MockConnectionBase {
@@ -68,7 +69,7 @@ class SharedConnectionEngine : public RpcEngine {
protected:
std::shared_ptr<RpcConnection> NewConnection() override {
// Stuff in some dummy endpoints so we don't error out
- last_endpoints_ = make_endpoint();
+ last_endpoints_ = make_endpoint()[0].endpoints;
return std::make_shared<RpcConnectionImpl<SharedMockRPCConnection>>(this);
}
@@ -180,6 +181,11 @@ TEST(RpcEngineTest, TestConnectionResetAndRecover) {
options.rpc_retry_delay_ms = 0;
SharedConnectionEngine engine(&io_service, options, "foo", "", "protocol", 1);
+ // Normally determined during RpcEngine::Connect, but in this case options
+ // provides enough info to determine policy here.
+ engine.TEST_SetRetryPolicy(engine.TEST_GenerateRetryPolicyUsingOptions());
+
+
EchoResponseProto server_resp;
server_resp.set_message("foo");
@@ -215,6 +221,10 @@ TEST(RpcEngineTest, TestConnectionResetAndRecoverWithDelay) {
options.rpc_retry_delay_ms = 1;
SharedConnectionEngine engine(&io_service, options, "foo", "", "protocol", 1);
+ // Normally determined during RpcEngine::Connect, but in this case options
+ // provides enough info to determine policy here.
+ engine.TEST_SetRetryPolicy(engine.TEST_GenerateRetryPolicyUsingOptions());
+
EchoResponseProto server_resp;
server_resp.set_message("foo");
@@ -339,6 +349,10 @@ TEST(RpcEngineTest, TestEventCallbacks)
options.rpc_retry_delay_ms = 0;
SharedConnectionEngine engine(&io_service, options, "foo", "", "protocol", 1);
+ // Normally determined during RpcEngine::Connect, but in this case options
+ // provides enough info to determine policy here.
+ engine.TEST_SetRetryPolicy(engine.TEST_GenerateRetryPolicyUsingOptions());
+
// Set up event callbacks
int calls = 0;
std::vector<std::string> callbacks;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/52] [abbrv] hadoop git commit: HDFS-11449: libhdfs++: Log
Datanode read size when reading from an HDFS block. Contributed by Xiaowei
Zhu.
Posted by jh...@apache.org.
HDFS-11449: libhdfs++: Log Datanode read size when reading from an HDFS block. Contributed by Xiaowei Zhu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0f3f8db1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0f3f8db1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0f3f8db1
Branch: refs/heads/HDFS-8707
Commit: 0f3f8db113621e177dbb7fc3e93f4870bff2d739
Parents: 606d20c
Author: James <jh...@apache.org>
Authored: Fri Feb 24 10:46:50 2017 -0500
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../src/main/native/libhdfspp/lib/fs/filehandle.cc | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f3f8db1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
index 1f553a0..24e81b6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
@@ -227,14 +227,15 @@ void FileHandleImpl::AsyncPreadSome(
std::string dnIpAddr = chosen_dn.id().ipaddr();
std::string dnHostName = chosen_dn.id().hostname();
- LOG_DEBUG(kFileHandle, << "FileHandleImpl::AsyncPreadSome("
- << FMT_THIS_ADDR << "), ...) Datanode hostname=" << dnHostName << ", IP Address=" << dnIpAddr
- << ", file path=\"" << path_ << "\", offset=" << offset);
uint64_t offset_within_block = offset - block->offset();
uint64_t size_within_block = std::min<uint64_t>(
block->b().numbytes() - offset_within_block, asio::buffer_size(buffers));
+ LOG_DEBUG(kFileHandle, << "FileHandleImpl::AsyncPreadSome("
+ << FMT_THIS_ADDR << "), ...) Datanode hostname=" << dnHostName << ", IP Address=" << dnIpAddr
+ << ", file path=\"" << path_ << "\", offset=" << offset << ", read size=" << size_within_block);
+
// This is where we will put the logic for re-using a DN connection; we can
// steal the FileHandle's dn and put it back when we're done
std::shared_ptr<DataNodeConnection> dn = CreateDataNodeConnection(io_service_, chosen_dn, &block->blocktoken());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/52] [abbrv] hadoop git commit: HDFS-11106: libhdfs++: Some
refactoring to better organize files (part 2). Contributed by James
Clampffer.
Posted by jh...@apache.org.
HDFS-11106: libhdfs++: Some refactoring to better organize files (part 2). Contributed by James Clampffer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3e53da2d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3e53da2d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3e53da2d
Branch: refs/heads/HDFS-8707
Commit: 3e53da2d623cfecc9f6af0fcf39413edb46b4092
Parents: 0f3f8db
Author: James <jh...@apache.org>
Authored: Mon Mar 6 12:30:19 2017 -0500
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../native/libhdfspp/lib/rpc/CMakeLists.txt | 2 +-
.../libhdfspp/lib/rpc/namenode_tracker.cc | 134 +++++
.../native/libhdfspp/lib/rpc/namenode_tracker.h | 81 +++
.../main/native/libhdfspp/lib/rpc/request.cc | 190 +++++++
.../src/main/native/libhdfspp/lib/rpc/request.h | 87 +++
.../native/libhdfspp/lib/rpc/rpc_connection.cc | 563 -------------------
.../native/libhdfspp/lib/rpc/rpc_connection.h | 552 +++++-------------
.../libhdfspp/lib/rpc/rpc_connection_impl.cc | 446 +++++++++++++++
.../libhdfspp/lib/rpc/rpc_connection_impl.h | 445 +++++++++++++++
.../main/native/libhdfspp/lib/rpc/rpc_engine.cc | 125 +---
.../main/native/libhdfspp/lib/rpc/rpc_engine.h | 250 +-------
.../native/libhdfspp/lib/rpc/sasl_protocol.cc | 1 +
.../native/libhdfspp/tests/rpc_engine_test.cc | 2 +-
13 files changed, 1535 insertions(+), 1343 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e53da2d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt
index 84debdd..e5a26fb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt
@@ -16,7 +16,7 @@
# limitations under the License.
#
-list(APPEND rpc_object_items rpc_connection.cc rpc_engine.cc sasl_protocol.cc sasl_engine.cc)
+list(APPEND rpc_object_items rpc_connection_impl.cc rpc_engine.cc namenode_tracker.cc request.cc sasl_protocol.cc sasl_engine.cc)
if (CMAKE_USING_CYRUS_SASL)
list(APPEND rpc_object_items cyrus_sasl_engine.cc)
endif (CMAKE_USING_CYRUS_SASL)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e53da2d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.cc
new file mode 100644
index 0000000..9d9a816
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.cc
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "namenode_tracker.h"
+
+#include "common/logging.h"
+#include "common/libhdfs_events_impl.h"
+#include "common/util.h"
+
+namespace hdfs {
+
+static std::string format_endpoints(const std::vector<::asio::ip::tcp::endpoint> &pts) {
+ std::stringstream ss;
+ for(unsigned int i=0; i<pts.size(); i++)
+ if(i == pts.size() - 1)
+ ss << pts[i];
+ else
+ ss << pts[i] << ", ";
+ return ss.str();
+}
+
+HANamenodeTracker::HANamenodeTracker(const std::vector<ResolvedNamenodeInfo> &servers,
+ ::asio::io_service *ioservice,
+ std::shared_ptr<LibhdfsEvents> event_handlers)
+ : enabled_(false), resolved_(false),
+ ioservice_(ioservice), event_handlers_(event_handlers)
+{
+ LOG_TRACE(kRPC, << "HANamenodeTracker got the following nodes");
+ for(unsigned int i=0;i<servers.size();i++)
+ LOG_TRACE(kRPC, << servers[i].str());
+
+ if(servers.size() >= 2) {
+ LOG_TRACE(kRPC, << "Creating HA namenode tracker");
+ if(servers.size() > 2) {
+ LOG_WARN(kRPC, << "Nameservice declares more than two nodes. Some won't be used.");
+ }
+
+ active_info_ = servers[0];
+ standby_info_ = servers[1];
+ LOG_INFO(kRPC, << "Active namenode url = " << active_info_.uri.str());
+ LOG_INFO(kRPC, << "Standby namenode url = " << standby_info_.uri.str());
+
+ enabled_ = true;
+ if(!active_info_.endpoints.empty() || !standby_info_.endpoints.empty()) {
+ resolved_ = true;
+ }
+ }
+}
+
+HANamenodeTracker::~HANamenodeTracker() {}
+
+// Pass in endpoint from current connection, this will do a reverse lookup
+// and return the info for the standby node. It will also swap its state internally.
+ResolvedNamenodeInfo HANamenodeTracker::GetFailoverAndUpdate(::asio::ip::tcp::endpoint current_endpoint) {
+ LOG_TRACE(kRPC, << "Swapping from endpoint " << current_endpoint);
+ mutex_guard swap_lock(swap_lock_);
+
+ ResolvedNamenodeInfo failover_node;
+
+ // Connected to standby, switch standby to active
+ if(IsCurrentActive_locked(current_endpoint)) {
+ std::swap(active_info_, standby_info_);
+ if(event_handlers_)
+ event_handlers_->call(FS_NN_FAILOVER_EVENT, active_info_.nameservice.c_str(),
+ reinterpret_cast<int64_t>(active_info_.uri.str().c_str()));
+ failover_node = active_info_;
+ } else if(IsCurrentStandby_locked(current_endpoint)) {
+ // Connected to standby
+ if(event_handlers_)
+ event_handlers_->call(FS_NN_FAILOVER_EVENT, active_info_.nameservice.c_str(),
+ reinterpret_cast<int64_t>(active_info_.uri.str().c_str()));
+ failover_node = active_info_;
+ } else {
+ // Invalid state, throw for testing
+ std::string ep1 = format_endpoints(active_info_.endpoints);
+ std::string ep2 = format_endpoints(standby_info_.endpoints);
+
+ std::stringstream msg;
+ msg << "Looked for " << current_endpoint << " in\n";
+ msg << ep1 << " and\n";
+ msg << ep2 << std::endl;
+
+ LOG_ERROR(kRPC, << "Unable to find RPC connection in config " << msg.str() << ". Bailing out.");
+ throw std::runtime_error(msg.str());
+ }
+
+ if(failover_node.endpoints.empty()) {
+ LOG_WARN(kRPC, << "No endpoints for node " << failover_node.uri.str() << " attempting to resolve again");
+ if(!ResolveInPlace(ioservice_, failover_node)) {
+ LOG_ERROR(kRPC, << "Fallback endpoint resolution for node " << failover_node.uri.str()
+ << "failed. Please make sure your configuration is up to date.");
+ }
+ }
+ return failover_node;
+}
+
+bool HANamenodeTracker::IsCurrentActive_locked(const ::asio::ip::tcp::endpoint &ep) const {
+ for(unsigned int i=0;i<active_info_.endpoints.size();i++) {
+ if(ep.address() == active_info_.endpoints[i].address()) {
+ if(ep.port() != active_info_.endpoints[i].port())
+ LOG_WARN(kRPC, << "Port mismatch: " << ep << " vs " << active_info_.endpoints[i] << " trying anyway..");
+ return true;
+ }
+ }
+ return false;
+}
+
+bool HANamenodeTracker::IsCurrentStandby_locked(const ::asio::ip::tcp::endpoint &ep) const {
+ for(unsigned int i=0;i<standby_info_.endpoints.size();i++) {
+ if(ep.address() == standby_info_.endpoints[i].address()) {
+ if(ep.port() != standby_info_.endpoints[i].port())
+ LOG_WARN(kRPC, << "Port mismatch: " << ep << " vs " << standby_info_.endpoints[i] << " trying anyway..");
+ return true;
+ }
+ }
+ return false;
+}
+
+} // end namespace hdfs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e53da2d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.h
new file mode 100644
index 0000000..f51e13c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.h
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIB_RPC_NAMENODE_TRACKER_H
+#define LIB_RPC_NAMENODE_TRACKER_H
+
+#include "common/libhdfs_events_impl.h"
+#include "common/namenode_info.h"
+
+#include <asio/ip/tcp.hpp>
+
+#include <memory>
+#include <mutex>
+#include <vector>
+
+namespace hdfs {
+
+/*
+ * Tracker gives the RpcEngine a quick way to use an endpoint that just
+ * failed in order to lookup a set of endpoints for a failover node.
+ *
+ * Note: For now this only deals with 2 NameNodes, but that's the default
+ * anyway.
+ */
+class HANamenodeTracker {
+ public:
+ HANamenodeTracker(const std::vector<ResolvedNamenodeInfo> &servers,
+ ::asio::io_service *ioservice,
+ std::shared_ptr<LibhdfsEvents> event_handlers_);
+
+ virtual ~HANamenodeTracker();
+
+ bool is_enabled() const { return enabled_; }
+ bool is_resolved() const { return resolved_; }
+
+ // Get node opposite of the current one if possible (swaps active/standby)
+ // Note: This will always mutate internal state. Use IsCurrentActive/Standby to
+ // get info without changing state
+ ResolvedNamenodeInfo GetFailoverAndUpdate(::asio::ip::tcp::endpoint current_endpoint);
+
+ bool IsCurrentActive_locked(const ::asio::ip::tcp::endpoint &ep) const;
+ bool IsCurrentStandby_locked(const ::asio::ip::tcp::endpoint &ep) const;
+
+ private:
+ // If HA should be enabled, according to our options and runtime info like # nodes provided
+ bool enabled_;
+ // If we were able to resolve at least 1 HA namenode
+ bool resolved_;
+
+ // Keep service in case a second round of DNS lookup is required
+ ::asio::io_service *ioservice_;
+
+ // Event handlers, for now this is the simplest place to catch all failover events
+ // and push info out to client application. Possibly move into RPCEngine.
+ std::shared_ptr<LibhdfsEvents> event_handlers_;
+
+ // Only support 1 active and 1 standby for now.
+ ResolvedNamenodeInfo active_info_;
+ ResolvedNamenodeInfo standby_info_;
+
+ // Aquire when switching from active-standby
+ std::mutex swap_lock_;
+};
+
+} // end namespace hdfs
+#endif // end include guard
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e53da2d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc
new file mode 100644
index 0000000..119962a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc
@@ -0,0 +1,190 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#include "request.h"
+#include "rpc_engine.h"
+#include "sasl_protocol.h"
+
+#include "RpcHeader.pb.h"
+#include "ProtobufRpcEngine.pb.h"
+#include "IpcConnectionContext.pb.h"
+
+#include <sstream>
+
+namespace hdfs {
+
+namespace pb = ::google::protobuf;
+namespace pbio = ::google::protobuf::io;
+
+using namespace ::hadoop::common;
+using namespace ::std::placeholders;
+
+static const int kNoRetry = -1;
+
+// Protobuf helper functions.
+static void AddHeadersToPacket(std::string *res,
+ std::initializer_list<const pb::MessageLite *> headers,
+ const std::string *payload) {
+ int len = 0;
+ std::for_each(
+ headers.begin(), headers.end(),
+ [&len](const pb::MessageLite *v) { len += DelimitedPBMessageSize(v); });
+
+ if (payload) {
+ len += payload->size();
+ }
+
+ int net_len = htonl(len);
+ res->reserve(res->size() + sizeof(net_len) + len);
+
+ pbio::StringOutputStream ss(res);
+ pbio::CodedOutputStream os(&ss);
+ os.WriteRaw(reinterpret_cast<const char *>(&net_len), sizeof(net_len));
+
+ uint8_t *buf = os.GetDirectBufferForNBytesAndAdvance(len);
+ assert(buf);
+
+ std::for_each(
+ headers.begin(), headers.end(), [&buf](const pb::MessageLite *v) {
+ buf = pbio::CodedOutputStream::WriteVarint32ToArray(v->ByteSize(), buf);
+ buf = v->SerializeWithCachedSizesToArray(buf);
+ });
+
+ if (payload) {
+ buf = os.WriteStringToArray(*payload, buf);
+ }
+}
+
+static void ConstructPayload(std::string *res, const pb::MessageLite *header) {
+ int len = DelimitedPBMessageSize(header);
+ res->reserve(len);
+ pbio::StringOutputStream ss(res);
+ pbio::CodedOutputStream os(&ss);
+ uint8_t *buf = os.GetDirectBufferForNBytesAndAdvance(len);
+ assert(buf);
+ buf = pbio::CodedOutputStream::WriteVarint32ToArray(header->ByteSize(), buf);
+ buf = header->SerializeWithCachedSizesToArray(buf);
+}
+
+static void ConstructPayload(std::string *res, const std::string *request) {
+ int len =
+ pbio::CodedOutputStream::VarintSize32(request->size()) + request->size();
+ res->reserve(len);
+ pbio::StringOutputStream ss(res);
+ pbio::CodedOutputStream os(&ss);
+ uint8_t *buf = os.GetDirectBufferForNBytesAndAdvance(len);
+ assert(buf);
+ buf = pbio::CodedOutputStream::WriteVarint32ToArray(request->size(), buf);
+ buf = os.WriteStringToArray(*request, buf);
+}
+
+static void SetRequestHeader(LockFreeRpcEngine *engine, int call_id,
+ const std::string &method_name, int retry_count,
+ RpcRequestHeaderProto *rpc_header,
+ RequestHeaderProto *req_header) {
+ rpc_header->set_rpckind(RPC_PROTOCOL_BUFFER);
+ rpc_header->set_rpcop(RpcRequestHeaderProto::RPC_FINAL_PACKET);
+ rpc_header->set_callid(call_id);
+ if (retry_count != kNoRetry)
+ rpc_header->set_retrycount(retry_count);
+ rpc_header->set_clientid(engine->client_id());
+
+ req_header->set_methodname(method_name);
+ req_header->set_declaringclassprotocolname(engine->protocol_name());
+ req_header->set_clientprotocolversion(engine->protocol_version());
+}
+
+// Request implementation
+
+Request::Request(LockFreeRpcEngine *engine, const std::string &method_name, int call_id,
+ const std::string &request, Handler &&handler)
+ : engine_(engine),
+ method_name_(method_name),
+ call_id_(call_id),
+ timer_(engine->io_service()),
+ handler_(std::move(handler)),
+ retry_count_(engine->retry_policy() ? 0 : kNoRetry),
+ failover_count_(0) {
+ ConstructPayload(&payload_, &request);
+}
+
+
+Request::Request(LockFreeRpcEngine *engine, const std::string &method_name, int call_id,
+ const pb::MessageLite *request, Handler &&handler)
+ : engine_(engine),
+ method_name_(method_name),
+ call_id_(call_id),
+ timer_(engine->io_service()),
+ handler_(std::move(handler)),
+ retry_count_(engine->retry_policy() ? 0 : kNoRetry),
+ failover_count_(0) {
+ ConstructPayload(&payload_, request);
+}
+
+Request::Request(LockFreeRpcEngine *engine, Handler &&handler)
+ : engine_(engine),
+ call_id_(-1),
+ timer_(engine->io_service()),
+ handler_(std::move(handler)),
+ retry_count_(engine->retry_policy() ? 0 : kNoRetry),
+ failover_count_(0) {
+}
+
+void Request::GetPacket(std::string *res) const {
+ LOG_TRACE(kRPC, << "Request::GetPacket called");
+
+ if (payload_.empty())
+ return;
+
+ RpcRequestHeaderProto rpc_header;
+ RequestHeaderProto req_header;
+ SetRequestHeader(engine_, call_id_, method_name_, retry_count_, &rpc_header,
+ &req_header);
+
+ // SASL messages don't have a request header
+ if (method_name_ != SASL_METHOD_NAME)
+ AddHeadersToPacket(res, {&rpc_header, &req_header}, &payload_);
+ else
+ AddHeadersToPacket(res, {&rpc_header}, &payload_);
+}
+
+void Request::OnResponseArrived(pbio::CodedInputStream *is,
+ const Status &status) {
+ LOG_TRACE(kRPC, << "Request::OnResponseArrived called");
+ handler_(is, status);
+}
+
+std::string Request::GetDebugString() const {
+ // Basic description of this object, aimed at debugging
+ std::stringstream ss;
+ ss << "\nRequest Object:\n";
+ ss << "\tMethod name = \"" << method_name_ << "\"\n";
+ ss << "\tCall id = " << call_id_ << "\n";
+ ss << "\tRetry Count = " << retry_count_ << "\n";
+ ss << "\tFailover count = " << failover_count_ << "\n";
+ return ss.str();
+}
+
+int Request::IncrementFailoverCount() {
+ // reset retry count when failing over
+ retry_count_ = 0;
+ return failover_count_++;
+}
+
+} // end namespace hdfs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e53da2d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.h
new file mode 100644
index 0000000..d265a4c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.h
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef LIB_RPC_RPC_REQUEST_H
+#define LIB_RPC_RPC_REQUEST_H
+
+#include "hdfspp/status.h"
+#include "common/util.h"
+#include "common/new_delete.h"
+
+#include <string>
+
+#include <google/protobuf/message_lite.h>
+#include <google/protobuf/io/coded_stream.h>
+#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
+
+#include <asio/deadline_timer.hpp>
+
+
+namespace hdfs {
+
+class LockFreeRpcEngine;
+class SaslProtocol;
+
+/*
+ * Internal bookkeeping for an outstanding request from the consumer.
+ *
+ * Threading model: not thread-safe; should only be accessed from a single
+ * thread at a time
+ */
+class Request {
+ public:
+ MEMCHECKED_CLASS(Request)
+ typedef std::function<void(::google::protobuf::io::CodedInputStream *is,
+ const Status &status)> Handler;
+
+ Request(LockFreeRpcEngine *engine, const std::string &method_name, int call_id,
+ const std::string &request, Handler &&callback);
+ Request(LockFreeRpcEngine *engine, const std::string &method_name, int call_id,
+ const ::google::protobuf::MessageLite *request, Handler &&callback);
+
+ // Null request (with no actual message) used to track the state of an
+ // initial Connect call
+ Request(LockFreeRpcEngine *engine, Handler &&handler);
+
+ int call_id() const { return call_id_; }
+ std::string method_name() const { return method_name_; }
+ ::asio::deadline_timer &timer() { return timer_; }
+ int IncrementRetryCount() { return retry_count_++; }
+ int IncrementFailoverCount();
+ void GetPacket(std::string *res) const;
+ void OnResponseArrived(::google::protobuf::io::CodedInputStream *is,
+ const Status &status);
+
+ int get_failover_count() {return failover_count_;}
+
+ std::string GetDebugString() const;
+
+ private:
+ LockFreeRpcEngine *const engine_;
+ const std::string method_name_;
+ const int call_id_;
+
+ ::asio::deadline_timer timer_;
+ std::string payload_;
+ const Handler handler_;
+
+ int retry_count_;
+ int failover_count_;
+};
+
+} // end namespace hdfs
+#endif // end include guard
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e53da2d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
deleted file mode 100644
index f629d1f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
+++ /dev/null
@@ -1,563 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include "rpc_engine.h"
-#include "sasl_protocol.h"
-
-#include "RpcHeader.pb.h"
-#include "ProtobufRpcEngine.pb.h"
-#include "IpcConnectionContext.pb.h"
-
-#include "common/logging.h"
-#include "common/util.h"
-
-#include <asio/read.hpp>
-
-namespace hdfs {
-
-namespace pb = ::google::protobuf;
-namespace pbio = ::google::protobuf::io;
-
-using namespace ::hadoop::common;
-using namespace ::std::placeholders;
-
-static const int kNoRetry = -1;
-
-static void AddHeadersToPacket(
- std::string *res, std::initializer_list<const pb::MessageLite *> headers,
- const std::string *payload) {
- int len = 0;
- std::for_each(
- headers.begin(), headers.end(),
- [&len](const pb::MessageLite *v) { len += DelimitedPBMessageSize(v); });
-
- if (payload) {
- len += payload->size();
- }
-
- int net_len = htonl(len);
- res->reserve(res->size() + sizeof(net_len) + len);
-
- pbio::StringOutputStream ss(res);
- pbio::CodedOutputStream os(&ss);
- os.WriteRaw(reinterpret_cast<const char *>(&net_len), sizeof(net_len));
-
- uint8_t *buf = os.GetDirectBufferForNBytesAndAdvance(len);
- assert(buf);
-
- std::for_each(
- headers.begin(), headers.end(), [&buf](const pb::MessageLite *v) {
- buf = pbio::CodedOutputStream::WriteVarint32ToArray(v->ByteSize(), buf);
- buf = v->SerializeWithCachedSizesToArray(buf);
- });
-
- if (payload) {
- buf = os.WriteStringToArray(*payload, buf);
- }
-}
-
-static void ConstructPayload(std::string *res, const pb::MessageLite *header) {
- int len = DelimitedPBMessageSize(header);
- res->reserve(len);
- pbio::StringOutputStream ss(res);
- pbio::CodedOutputStream os(&ss);
- uint8_t *buf = os.GetDirectBufferForNBytesAndAdvance(len);
- assert(buf);
- buf = pbio::CodedOutputStream::WriteVarint32ToArray(header->ByteSize(), buf);
- buf = header->SerializeWithCachedSizesToArray(buf);
-}
-
-static void ConstructPayload(std::string *res, const std::string *request) {
- int len =
- pbio::CodedOutputStream::VarintSize32(request->size()) + request->size();
- res->reserve(len);
- pbio::StringOutputStream ss(res);
- pbio::CodedOutputStream os(&ss);
- uint8_t *buf = os.GetDirectBufferForNBytesAndAdvance(len);
- assert(buf);
- buf = pbio::CodedOutputStream::WriteVarint32ToArray(request->size(), buf);
- buf = os.WriteStringToArray(*request, buf);
-}
-
-static void SetRequestHeader(LockFreeRpcEngine *engine, int call_id,
- const std::string &method_name, int retry_count,
- RpcRequestHeaderProto *rpc_header,
- RequestHeaderProto *req_header) {
- rpc_header->set_rpckind(RPC_PROTOCOL_BUFFER);
- rpc_header->set_rpcop(RpcRequestHeaderProto::RPC_FINAL_PACKET);
- rpc_header->set_callid(call_id);
- if (retry_count != kNoRetry)
- rpc_header->set_retrycount(retry_count);
- rpc_header->set_clientid(engine->client_id());
-
- req_header->set_methodname(method_name);
- req_header->set_declaringclassprotocolname(engine->protocol_name());
- req_header->set_clientprotocolversion(engine->protocol_version());
-}
-
-RpcConnection::~RpcConnection() {}
-
-Request::Request(LockFreeRpcEngine *engine, const std::string &method_name, int call_id,
- const std::string &request, Handler &&handler)
- : engine_(engine),
- method_name_(method_name),
- call_id_(call_id),
- timer_(engine->io_service()),
- handler_(std::move(handler)),
- retry_count_(engine->retry_policy() ? 0 : kNoRetry),
- failover_count_(0) {
- ConstructPayload(&payload_, &request);
-}
-
-Request::Request(LockFreeRpcEngine *engine, const std::string &method_name, int call_id,
- const pb::MessageLite *request, Handler &&handler)
- : engine_(engine),
- method_name_(method_name),
- call_id_(call_id),
- timer_(engine->io_service()),
- handler_(std::move(handler)),
- retry_count_(engine->retry_policy() ? 0 : kNoRetry),
- failover_count_(0) {
- ConstructPayload(&payload_, request);
-}
-
-Request::Request(LockFreeRpcEngine *engine, Handler &&handler)
- : engine_(engine),
- call_id_(-1),
- timer_(engine->io_service()),
- handler_(std::move(handler)),
- retry_count_(engine->retry_policy() ? 0 : kNoRetry),
- failover_count_(0) {
-}
-
-void Request::GetPacket(std::string *res) const {
- LOG_TRACE(kRPC, << "Request::GetPacket called");
-
- if (payload_.empty())
- return;
-
- RpcRequestHeaderProto rpc_header;
- RequestHeaderProto req_header;
- SetRequestHeader(engine_, call_id_, method_name_, retry_count_, &rpc_header,
- &req_header);
-
- // SASL messages don't have a request header
- if (method_name_ != SASL_METHOD_NAME)
- AddHeadersToPacket(res, {&rpc_header, &req_header}, &payload_);
- else
- AddHeadersToPacket(res, {&rpc_header}, &payload_);
-}
-
-void Request::OnResponseArrived(pbio::CodedInputStream *is,
- const Status &status) {
- LOG_TRACE(kRPC, << "Request::OnResponseArrived called");
- handler_(is, status);
-}
-
-std::string Request::GetDebugString() const {
- // Basic description of this object, aimed at debugging
- std::stringstream ss;
- ss << "\nRequest Object:\n";
- ss << "\tMethod name = \"" << method_name_ << "\"\n";
- ss << "\tCall id = " << call_id_ << "\n";
- ss << "\tRetry Count = " << retry_count_ << "\n";
- ss << "\tFailover count = " << failover_count_ << "\n";
- return ss.str();
-}
-
-int Request::IncrementFailoverCount() {
- // reset retry count when failing over
- retry_count_ = 0;
- return failover_count_++;
-}
-
-RpcConnection::RpcConnection(LockFreeRpcEngine *engine)
- : engine_(engine),
- connected_(kNotYetConnected) {}
-
-::asio::io_service &RpcConnection::io_service() {
- return engine_->io_service();
-}
-
-void RpcConnection::StartReading() {
- auto shared_this = shared_from_this();
- io_service().post([shared_this, this] () {
- OnRecvCompleted(::asio::error_code(), 0);
- });
-}
-
-void RpcConnection::HandshakeComplete(const Status &s) {
- std::lock_guard<std::mutex> state_lock(connection_state_lock_);
-
- LOG_TRACE(kRPC, << "RpcConnectionImpl::HandshakeComplete called");
-
- if (s.ok()) {
- if (connected_ == kHandshaking) {
- auto shared_this = shared_from_this();
-
- connected_ = kAuthenticating;
- if (auth_info_.useSASL()) {
-#ifdef USE_SASL
- sasl_protocol_ = std::make_shared<SaslProtocol>(cluster_name_, auth_info_, shared_from_this());
- sasl_protocol_->SetEventHandlers(event_handlers_);
- sasl_protocol_->Authenticate([shared_this, this](
- const Status & status, const AuthInfo & new_auth_info) {
- AuthComplete(status, new_auth_info); } );
-#else
- AuthComplete_locked(Status::Error("SASL is required, but no SASL library was found"), auth_info_);
-#endif
- } else {
- AuthComplete_locked(Status::OK(), auth_info_);
- }
- }
- } else {
- CommsError(s);
- };
-}
-
-void RpcConnection::AuthComplete(const Status &s, const AuthInfo & new_auth_info) {
- std::lock_guard<std::mutex> state_lock(connection_state_lock_);
- AuthComplete_locked(s, new_auth_info);
-}
-
-void RpcConnection::AuthComplete_locked(const Status &s, const AuthInfo & new_auth_info) {
- assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
- LOG_TRACE(kRPC, << "RpcConnectionImpl::AuthComplete called");
-
- // Free the sasl_protocol object
- sasl_protocol_.reset();
-
- if (s.ok()) {
- auth_info_ = new_auth_info;
-
- auto shared_this = shared_from_this();
- SendContext([shared_this, this](const Status & s) {
- ContextComplete(s);
- });
- } else {
- CommsError(s);
- };
-}
-
-void RpcConnection::ContextComplete(const Status &s) {
- std::lock_guard<std::mutex> state_lock(connection_state_lock_);
-
- LOG_TRACE(kRPC, << "RpcConnectionImpl::ContextComplete called");
-
- if (s.ok()) {
- if (connected_ == kAuthenticating) {
- connected_ = kConnected;
- }
- FlushPendingRequests();
- } else {
- CommsError(s);
- };
-}
-
-void RpcConnection::AsyncFlushPendingRequests() {
- std::shared_ptr<RpcConnection> shared_this = shared_from_this();
- io_service().post([shared_this, this]() {
- std::lock_guard<std::mutex> state_lock(connection_state_lock_);
-
- LOG_TRACE(kRPC, << "RpcConnection::AsyncFlushPendingRequests called (connected=" << ToString(connected_) << ")");
-
- if (!request_over_the_wire_) {
- FlushPendingRequests();
- }
- });
-}
-
-Status RpcConnection::HandleRpcResponse(std::shared_ptr<Response> response) {
- assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
-
- response->ar.reset(new pbio::ArrayInputStream(&response->data_[0], response->data_.size()));
- response->in.reset(new pbio::CodedInputStream(response->ar.get()));
- response->in->PushLimit(response->data_.size());
- RpcResponseHeaderProto h;
- ReadDelimitedPBMessage(response->in.get(), &h);
-
- auto req = RemoveFromRunningQueue(h.callid());
- if (!req) {
- LOG_WARN(kRPC, << "RPC response with Unknown call id " << h.callid());
- if((int32_t)h.callid() == RpcEngine::kCallIdSasl) {
- return Status::AuthenticationFailed("You have an unsecured client connecting to a secured server");
- } else {
- return Status::Error("Rpc response with unknown call id");
- }
- }
-
- Status status;
- if(event_handlers_) {
- event_response event_resp = event_handlers_->call(FS_NN_READ_EVENT, cluster_name_.c_str(), 0);
-#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
- if (event_resp.response() == event_response::kTest_Error) {
- status = event_resp.status();
- }
-#endif
- }
-
- if (status.ok() && h.has_exceptionclassname()) {
- status =
- Status::Exception(h.exceptionclassname().c_str(), h.errormsg().c_str());
- }
-
- if(status.get_server_exception_type() == Status::kStandbyException) {
- LOG_WARN(kRPC, << "Tried to connect to standby. status = " << status.ToString());
-
- // We got the request back, but it needs to be resent to the other NN
- std::vector<std::shared_ptr<Request>> reqs_to_redirect = {req};
- PrependRequests_locked(reqs_to_redirect);
-
- CommsError(status);
- return status;
- }
-
- io_service().post([req, response, status]() {
- req->OnResponseArrived(response->in.get(), status); // Never call back while holding a lock
- });
-
- return Status::OK();
-}
-
-void RpcConnection::HandleRpcTimeout(std::shared_ptr<Request> req,
- const ::asio::error_code &ec) {
- if (ec.value() == asio::error::operation_aborted) {
- return;
- }
-
- std::lock_guard<std::mutex> state_lock(connection_state_lock_);
- auto r = RemoveFromRunningQueue(req->call_id());
- if (!r) {
- // The RPC might have been finished and removed from the queue
- return;
- }
-
- Status stat = ToStatus(ec ? ec : make_error_code(::asio::error::timed_out));
-
- r->OnResponseArrived(nullptr, stat);
-}
-
-std::shared_ptr<std::string> RpcConnection::PrepareHandshakePacket() {
- assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
-
- /** From Client.java:
- *
- * Write the connection header - this is sent when connection is established
- * +----------------------------------+
- * | "hrpc" 4 bytes |
- * +----------------------------------+
- * | Version (1 byte) |
- * +----------------------------------+
- * | Service Class (1 byte) |
- * +----------------------------------+
- * | AuthProtocol (1 byte) |
- * +----------------------------------+
- *
- * AuthProtocol: 0->none, -33->SASL
- */
-
- char auth_protocol = auth_info_.useSASL() ? -33 : 0;
- const char handshake_header[] = {'h', 'r', 'p', 'c',
- RpcEngine::kRpcVersion, 0, auth_protocol};
- auto res =
- std::make_shared<std::string>(handshake_header, sizeof(handshake_header));
-
- return res;
-}
-
-std::shared_ptr<std::string> RpcConnection::PrepareContextPacket() {
- // This needs to be send after the SASL handshake, and
- // after the SASL handshake (if any)
- assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
-
- auto res = std::make_shared<std::string>();
-
- RpcRequestHeaderProto h;
- h.set_rpckind(RPC_PROTOCOL_BUFFER);
- h.set_rpcop(RpcRequestHeaderProto::RPC_FINAL_PACKET);
- h.set_callid(RpcEngine::kCallIdConnectionContext);
- h.set_clientid(engine_->client_name());
-
- IpcConnectionContextProto handshake;
- handshake.set_protocol(engine_->protocol_name());
- const std::string & user_name = auth_info_.getUser();
- if (!user_name.empty()) {
- *handshake.mutable_userinfo()->mutable_effectiveuser() = user_name;
- }
- AddHeadersToPacket(res.get(), {&h, &handshake}, nullptr);
-
- return res;
-}
-
-void RpcConnection::AsyncRpc(
- const std::string &method_name, const ::google::protobuf::MessageLite *req,
- std::shared_ptr<::google::protobuf::MessageLite> resp,
- const RpcCallback &handler) {
- std::lock_guard<std::mutex> state_lock(connection_state_lock_);
- AsyncRpc_locked(method_name, req, resp, handler);
-}
-
-void RpcConnection::AsyncRpc_locked(
- const std::string &method_name, const ::google::protobuf::MessageLite *req,
- std::shared_ptr<::google::protobuf::MessageLite> resp,
- const RpcCallback &handler) {
- assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
-
- auto wrapped_handler =
- [resp, handler](pbio::CodedInputStream *is, const Status &status) {
- if (status.ok()) {
- if (is) { // Connect messages will not have an is
- ReadDelimitedPBMessage(is, resp.get());
- }
- }
- handler(status);
- };
-
- int call_id = (method_name != SASL_METHOD_NAME ? engine_->NextCallId() : RpcEngine::kCallIdSasl);
- auto r = std::make_shared<Request>(engine_, method_name, call_id, req,
- std::move(wrapped_handler));
- auto r_vector = std::vector<std::shared_ptr<Request> > (1, r);
- SendRpcRequests(r_vector);
-}
-
-void RpcConnection::AsyncRpc(const std::vector<std::shared_ptr<Request> > & requests) {
- std::lock_guard<std::mutex> state_lock(connection_state_lock_);
- SendRpcRequests(requests);
-}
-
-void RpcConnection::SendRpcRequests(const std::vector<std::shared_ptr<Request> > & requests) {
- LOG_TRACE(kRPC, << "RpcConnection::SendRpcRequests[] called; connected=" << ToString(connected_));
- assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
-
- if (connected_ == kDisconnected) {
- // Oops. The connection failed _just_ before the engine got a chance
- // to send it. Register it as a failure
- Status status = Status::ResourceUnavailable("RpcConnection closed before send.");
- engine_->AsyncRpcCommsError(status, shared_from_this(), requests);
- } else {
- for (auto r: requests) {
- if (r->method_name() != SASL_METHOD_NAME)
- pending_requests_.push_back(r);
- else
- auth_requests_.push_back(r);
- }
- if (connected_ == kConnected || connected_ == kHandshaking || connected_ == kAuthenticating) { // Dont flush if we're waiting or handshaking
- FlushPendingRequests();
- }
- }
-}
-
-
-void RpcConnection::PreEnqueueRequests(
- std::vector<std::shared_ptr<Request>> requests) {
- // Public method - acquire lock
- std::lock_guard<std::mutex> state_lock(connection_state_lock_);
-
- LOG_DEBUG(kRPC, << "RpcConnection::PreEnqueueRequests called");
-
- assert(connected_ == kNotYetConnected);
-
- pending_requests_.insert(pending_requests_.end(), requests.begin(),
- requests.end());
- // Don't start sending yet; will flush when connected
-}
-
-// Only call when already holding conn state lock
-void RpcConnection::PrependRequests_locked( std::vector<std::shared_ptr<Request>> requests) {
- LOG_DEBUG(kRPC, << "RpcConnection::PrependRequests called");
-
- pending_requests_.insert(pending_requests_.begin(), requests.begin(),
- requests.end());
- // Don't start sending yet; will flush when connected
-}
-
-void RpcConnection::SetEventHandlers(std::shared_ptr<LibhdfsEvents> event_handlers) {
- std::lock_guard<std::mutex> state_lock(connection_state_lock_);
- event_handlers_ = event_handlers;
- if (sasl_protocol_) {
- sasl_protocol_->SetEventHandlers(event_handlers);
- }
-}
-
-void RpcConnection::SetClusterName(std::string cluster_name) {
- std::lock_guard<std::mutex> state_lock(connection_state_lock_);
- cluster_name_ = cluster_name;
-}
-
-void RpcConnection::CommsError(const Status &status) {
- assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
- LOG_DEBUG(kRPC, << "RpcConnection::CommsError called");
-
- Disconnect();
-
- // Anything that has been queued to the connection (on the fly or pending)
- // will get dinged for a retry
- std::vector<std::shared_ptr<Request>> requestsToReturn;
- std::transform(requests_on_fly_.begin(), requests_on_fly_.end(),
- std::back_inserter(requestsToReturn),
- std::bind(&RequestOnFlyMap::value_type::second, _1));
- requests_on_fly_.clear();
-
- requestsToReturn.insert(requestsToReturn.end(),
- std::make_move_iterator(pending_requests_.begin()),
- std::make_move_iterator(pending_requests_.end()));
- pending_requests_.clear();
-
- engine_->AsyncRpcCommsError(status, shared_from_this(), requestsToReturn);
-}
-
-void RpcConnection::ClearAndDisconnect(const ::asio::error_code &ec) {
- Disconnect();
- std::vector<std::shared_ptr<Request>> requests;
- std::transform(requests_on_fly_.begin(), requests_on_fly_.end(),
- std::back_inserter(requests),
- std::bind(&RequestOnFlyMap::value_type::second, _1));
- requests_on_fly_.clear();
- requests.insert(requests.end(),
- std::make_move_iterator(pending_requests_.begin()),
- std::make_move_iterator(pending_requests_.end()));
- pending_requests_.clear();
- for (const auto &req : requests) {
- req->OnResponseArrived(nullptr, ToStatus(ec));
- }
-}
-
-std::shared_ptr<Request> RpcConnection::RemoveFromRunningQueue(int call_id) {
- assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
- auto it = requests_on_fly_.find(call_id);
- if (it == requests_on_fly_.end()) {
- return std::shared_ptr<Request>();
- }
-
- auto req = it->second;
- requests_on_fly_.erase(it);
- return req;
-}
-
-std::string RpcConnection::ToString(ConnectedState connected) {
- switch(connected) {
- case kNotYetConnected: return "NotYetConnected";
- case kConnecting: return "Connecting";
- case kHandshaking: return "Handshaking";
- case kAuthenticating: return "Authenticating";
- case kConnected: return "Connected";
- case kDisconnected: return "Disconnected";
- default: return "Invalid ConnectedState";
- }
-}
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e53da2d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
index a6a07c4..7a671fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
@@ -15,430 +15,166 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-#ifndef LIB_RPC_RPC_CONNECTION_H_
-#define LIB_RPC_RPC_CONNECTION_H_
+#ifndef LIB_RPC_RPC_CONNECTION_H
+#define LIB_RPC_RPC_CONNECTION_H
-#include "rpc_engine.h"
+/*
+ * Encapsulates a persistent connection to the NameNode, and the sending of
+ * RPC requests and evaluating their responses.
+ *
+ * Can have multiple RPC requests in-flight simultaneously, but they are
+ * evaluated in-order on the server side in a blocking manner.
+ *
+ * Threading model: public interface is thread-safe
+ * All handlers passed in to method calls will be called from an asio thread,
+ * and will not be holding any internal RpcConnection locks.
+ */
+#include "request.h"
#include "common/auth_info.h"
-#include "common/logging.h"
-#include "common/util.h"
#include "common/libhdfs_events_impl.h"
-#include "sasl_protocol.h"
-
-#include <asio/connect.hpp>
-#include <asio/read.hpp>
-#include <asio/write.hpp>
+#include "common/new_delete.h"
+#include "hdfspp/status.h"
-#include <system_error>
+#include <functional>
+#include <memory>
+#include <vector>
+#include <deque>
+#include <unordered_map>
namespace hdfs {
-template <class Socket>
-class RpcConnectionImpl : public RpcConnection {
-public:
- MEMCHECKED_CLASS(RpcConnectionImpl);
+typedef const std::function<void(const Status &)> RpcCallback;
- RpcConnectionImpl(RpcEngine *engine);
- virtual ~RpcConnectionImpl() override;
+class LockFreeRpcEngine;
+class SaslProtocol;
+class RpcConnection : public std::enable_shared_from_this<RpcConnection> {
+ public:
+ MEMCHECKED_CLASS(RpcConnection)
+ RpcConnection(LockFreeRpcEngine *engine);
+ virtual ~RpcConnection();
+
+ // Note that a single server can have multiple endpoints - especially both
+ // an ipv4 and ipv6 endpoint
virtual void Connect(const std::vector<::asio::ip::tcp::endpoint> &server,
const AuthInfo & auth_info,
- RpcCallback &handler);
- virtual void ConnectAndFlush(
- const std::vector<::asio::ip::tcp::endpoint> &server) override;
- virtual void SendHandshake(RpcCallback &handler) override;
- virtual void SendContext(RpcCallback &handler) override;
- virtual void Disconnect() override;
- virtual void OnSendCompleted(const ::asio::error_code &ec,
- size_t transferred) override;
- virtual void OnRecvCompleted(const ::asio::error_code &ec,
- size_t transferred) override;
- virtual void FlushPendingRequests() override;
+ RpcCallback &handler) = 0;
+ virtual void ConnectAndFlush(const std::vector<::asio::ip::tcp::endpoint> &server) = 0;
+ virtual void Disconnect() = 0;
+
+ void StartReading();
+ void AsyncRpc(const std::string &method_name,
+ const ::google::protobuf::MessageLite *req,
+ std::shared_ptr<::google::protobuf::MessageLite> resp,
+ const RpcCallback &handler);
+ void AsyncRpc(const std::vector<std::shared_ptr<Request> > & requests);
- Socket &TEST_get_mutable_socket() { return socket_; }
+ // Enqueue requests before the connection is connected. Will be flushed
+ // on connect
+ void PreEnqueueRequests(std::vector<std::shared_ptr<Request>> requests);
- void TEST_set_connected(bool connected) { connected_ = connected ? kConnected : kNotYetConnected; }
+ // Put requests at the front of the current request queue
+ void PrependRequests_locked(std::vector<std::shared_ptr<Request>> requests);
- private:
- const Options options_;
- ::asio::ip::tcp::endpoint current_endpoint_;
- std::vector<::asio::ip::tcp::endpoint> additional_endpoints_;
- Socket socket_;
- ::asio::deadline_timer connect_timer_;
+ void SetEventHandlers(std::shared_ptr<LibhdfsEvents> event_handlers);
+ void SetClusterName(std::string cluster_name);
- void ConnectComplete(const ::asio::error_code &ec, const ::asio::ip::tcp::endpoint &remote);
+ LockFreeRpcEngine *engine() { return engine_; }
+ ::asio::io_service &io_service();
+
+ protected:
+ struct Response {
+ enum ResponseState {
+ kReadLength,
+ kReadContent,
+ kParseResponse,
+ } state_;
+ unsigned length_;
+ std::vector<char> data_;
+
+ std::unique_ptr<::google::protobuf::io::ArrayInputStream> ar;
+ std::unique_ptr<::google::protobuf::io::CodedInputStream> in;
+
+ Response() : state_(kReadLength), length_(0) {}
+ };
+
+
+ // Initial handshaking protocol: connect->handshake-->(auth)?-->context->connected
+ virtual void SendHandshake(RpcCallback &handler) = 0;
+ void HandshakeComplete(const Status &s);
+ void AuthComplete(const Status &s, const AuthInfo & new_auth_info);
+ void AuthComplete_locked(const Status &s, const AuthInfo & new_auth_info);
+ virtual void SendContext(RpcCallback &handler) = 0;
+ void ContextComplete(const Status &s);
+
+ virtual void OnSendCompleted(const ::asio::error_code &ec,
+ size_t transferred) = 0;
+ virtual void OnRecvCompleted(const ::asio::error_code &ec,
+ size_t transferred) = 0;
+ virtual void FlushPendingRequests()=0; // Synchronously write the next request
+
+ void AsyncRpc_locked(
+ const std::string &method_name,
+ const ::google::protobuf::MessageLite *req,
+ std::shared_ptr<::google::protobuf::MessageLite> resp,
+ const RpcCallback &handler);
+ void SendRpcRequests(const std::vector<std::shared_ptr<Request> > & requests);
+ void AsyncFlushPendingRequests(); // Queue requests to be flushed at a later time
+
+
+
+ std::shared_ptr<std::string> PrepareHandshakePacket();
+ std::shared_ptr<std::string> PrepareContextPacket();
+ static std::string SerializeRpcRequest(const std::string &method_name,
+ const ::google::protobuf::MessageLite *req);
+
+ Status HandleRpcResponse(std::shared_ptr<Response> response);
+ void HandleRpcTimeout(std::shared_ptr<Request> req,
+ const ::asio::error_code &ec);
+ void CommsError(const Status &status);
+
+ void ClearAndDisconnect(const ::asio::error_code &ec);
+ std::shared_ptr<Request> RemoveFromRunningQueue(int call_id);
+
+ LockFreeRpcEngine *const engine_;
+ std::shared_ptr<Response> current_response_state_;
+ AuthInfo auth_info_;
+
+ // Connection can have deferred connection, especially when we're pausing
+ // during retry
+ enum ConnectedState {
+ kNotYetConnected,
+ kConnecting,
+ kHandshaking,
+ kAuthenticating,
+ kConnected,
+ kDisconnected
+ };
+ static std::string ToString(ConnectedState connected);
+ ConnectedState connected_;
+
+ // State machine for performing a SASL handshake
+ std::shared_ptr<SaslProtocol> sasl_protocol_;
+ // The request being sent over the wire; will also be in requests_on_fly_
+ std::shared_ptr<Request> request_over_the_wire_;
+ // Requests to be sent over the wire
+ std::deque<std::shared_ptr<Request>> pending_requests_;
+ // Requests to be sent over the wire during authentication; not retried if
+ // there is a connection error
+ std::deque<std::shared_ptr<Request>> auth_requests_;
+ // Requests that are waiting for responses
+ typedef std::unordered_map<int, std::shared_ptr<Request>> RequestOnFlyMap;
+ RequestOnFlyMap requests_on_fly_;
+ std::shared_ptr<LibhdfsEvents> event_handlers_;
+ std::string cluster_name_;
+
+ // Lock for mutable parts of this class that need to be thread safe
+ std::mutex connection_state_lock_;
+
+ friend class SaslProtocol;
};
-template <class Socket>
-RpcConnectionImpl<Socket>::RpcConnectionImpl(RpcEngine *engine)
- : RpcConnection(engine),
- options_(engine->options()),
- socket_(engine->io_service()),
- connect_timer_(engine->io_service())
-{
- LOG_TRACE(kRPC, << "RpcConnectionImpl::RpcConnectionImpl called &" << (void*)this);
-}
-
-template <class Socket>
-RpcConnectionImpl<Socket>::~RpcConnectionImpl() {
- LOG_DEBUG(kRPC, << "RpcConnectionImpl::~RpcConnectionImpl called &" << (void*)this);
-
- if (pending_requests_.size() > 0)
- LOG_WARN(kRPC, << "RpcConnectionImpl::~RpcConnectionImpl called with items in the pending queue");
- if (requests_on_fly_.size() > 0)
- LOG_WARN(kRPC, << "RpcConnectionImpl::~RpcConnectionImpl called with items in the requests_on_fly queue");
-}
-
-template <class Socket>
-void RpcConnectionImpl<Socket>::Connect(
- const std::vector<::asio::ip::tcp::endpoint> &server,
- const AuthInfo & auth_info,
- RpcCallback &handler) {
- LOG_TRACE(kRPC, << "RpcConnectionImpl::Connect called");
-
- this->auth_info_ = auth_info;
-
- auto connectionSuccessfulReq = std::make_shared<Request>(
- engine_, [handler](::google::protobuf::io::CodedInputStream *is,
- const Status &status) {
- (void)is;
- handler(status);
- });
- pending_requests_.push_back(connectionSuccessfulReq);
- this->ConnectAndFlush(server); // need "this" so compiler can infer type of CAF
-}
-
-template <class Socket>
-void RpcConnectionImpl<Socket>::ConnectAndFlush(
- const std::vector<::asio::ip::tcp::endpoint> &server) {
-
- LOG_INFO(kRPC, << "ConnectAndFlush called");
- std::lock_guard<std::mutex> state_lock(connection_state_lock_);
-
- if (server.empty()) {
- Status s = Status::InvalidArgument("No endpoints provided");
- CommsError(s);
- return;
- }
-
- if (connected_ == kConnected) {
- FlushPendingRequests();
- return;
- }
- if (connected_ != kNotYetConnected) {
- LOG_WARN(kRPC, << "RpcConnectionImpl::ConnectAndFlush called while connected=" << ToString(connected_));
- return;
- }
- connected_ = kConnecting;
-
- // Take the first endpoint, but remember the alternatives for later
- additional_endpoints_ = server;
- ::asio::ip::tcp::endpoint first_endpoint = additional_endpoints_.front();
- additional_endpoints_.erase(additional_endpoints_.begin());
- current_endpoint_ = first_endpoint;
-
- auto shared_this = shared_from_this();
- socket_.async_connect(first_endpoint, [shared_this, this, first_endpoint](const ::asio::error_code &ec) {
- ConnectComplete(ec, first_endpoint);
- });
-
- // Prompt the timer to timeout
- auto weak_this = std::weak_ptr<RpcConnection>(shared_this);
- connect_timer_.expires_from_now(
- std::chrono::milliseconds(options_.rpc_connect_timeout));
- connect_timer_.async_wait([shared_this, this, first_endpoint](const ::asio::error_code &ec) {
- if (ec)
- ConnectComplete(ec, first_endpoint);
- else
- ConnectComplete(make_error_code(asio::error::host_unreachable), first_endpoint);
- });
-}
-
-template <class Socket>
-void RpcConnectionImpl<Socket>::ConnectComplete(const ::asio::error_code &ec, const ::asio::ip::tcp::endpoint & remote) {
- auto shared_this = RpcConnectionImpl<Socket>::shared_from_this();
- std::lock_guard<std::mutex> state_lock(connection_state_lock_);
- connect_timer_.cancel();
-
- LOG_TRACE(kRPC, << "RpcConnectionImpl::ConnectComplete called");
-
- // Could be an old async connect returning a result after we've moved on
- if (remote != current_endpoint_) {
- LOG_DEBUG(kRPC, << "Got ConnectComplete for " << remote << " but current_endpoint_ is " << current_endpoint_);
- return;
- }
- if (connected_ != kConnecting) {
- LOG_DEBUG(kRPC, << "Got ConnectComplete but current state is " << connected_);;
- return;
- }
-
- Status status = ToStatus(ec);
- if(event_handlers_) {
- event_response event_resp = event_handlers_->call(FS_NN_CONNECT_EVENT, cluster_name_.c_str(), 0);
-#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
- if (event_resp.response() == event_response::kTest_Error) {
- status = event_resp.status();
- }
-#endif
- }
-
- if (status.ok()) {
- StartReading();
- SendHandshake([shared_this, this](const Status & s) {
- HandshakeComplete(s);
- });
- } else {
- LOG_DEBUG(kRPC, << "Rpc connection failed; err=" << status.ToString());;
- std::string err = SafeDisconnect(get_asio_socket_ptr(&socket_));
- if(!err.empty()) {
- LOG_INFO(kRPC, << "Rpc connection failed to connect to endpoint, error closing connection: " << err);
- }
-
- if (!additional_endpoints_.empty()) {
- // If we have additional endpoints, keep trying until we either run out or
- // hit one
- ::asio::ip::tcp::endpoint next_endpoint = additional_endpoints_.front();
- additional_endpoints_.erase(additional_endpoints_.begin());
- current_endpoint_ = next_endpoint;
-
- socket_.async_connect(next_endpoint, [shared_this, this, next_endpoint](const ::asio::error_code &ec) {
- ConnectComplete(ec, next_endpoint);
- });
- connect_timer_.expires_from_now(
- std::chrono::milliseconds(options_.rpc_connect_timeout));
- connect_timer_.async_wait([shared_this, this, next_endpoint](const ::asio::error_code &ec) {
- if (ec)
- ConnectComplete(ec, next_endpoint);
- else
- ConnectComplete(make_error_code(asio::error::host_unreachable), next_endpoint);
- });
- } else {
- CommsError(status);
- }
- }
-}
-
-template <class Socket>
-void RpcConnectionImpl<Socket>::SendHandshake(RpcCallback &handler) {
- assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
-
- LOG_TRACE(kRPC, << "RpcConnectionImpl::SendHandshake called");
- connected_ = kHandshaking;
-
- auto shared_this = shared_from_this();
- auto handshake_packet = PrepareHandshakePacket();
- ::asio::async_write(socket_, asio::buffer(*handshake_packet),
- [handshake_packet, handler, shared_this, this](
- const ::asio::error_code &ec, size_t) {
- Status status = ToStatus(ec);
- handler(status);
- });
-}
-
-template <class Socket>
-void RpcConnectionImpl<Socket>::SendContext(RpcCallback &handler) {
- assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
-
- LOG_TRACE(kRPC, << "RpcConnectionImpl::SendContext called");
-
- auto shared_this = shared_from_this();
- auto context_packet = PrepareContextPacket();
- ::asio::async_write(socket_, asio::buffer(*context_packet),
- [context_packet, handler, shared_this, this](
- const ::asio::error_code &ec, size_t) {
- Status status = ToStatus(ec);
- handler(status);
- });
-}
-
-template <class Socket>
-void RpcConnectionImpl<Socket>::OnSendCompleted(const ::asio::error_code &ec,
- size_t) {
- using std::placeholders::_1;
- using std::placeholders::_2;
- std::lock_guard<std::mutex> state_lock(connection_state_lock_);
-
- LOG_TRACE(kRPC, << "RpcConnectionImpl::OnSendCompleted called");
-
- request_over_the_wire_.reset();
- if (ec) {
- LOG_WARN(kRPC, << "Network error during RPC write: " << ec.message());
- CommsError(ToStatus(ec));
- return;
- }
-
- FlushPendingRequests();
-}
-
-template <class Socket>
-void RpcConnectionImpl<Socket>::FlushPendingRequests() {
- using namespace ::std::placeholders;
-
- // Lock should be held
- assert(lock_held(connection_state_lock_));
-
- LOG_TRACE(kRPC, << "RpcConnectionImpl::FlushPendingRequests called");
-
- // Don't send if we don't need to
- if (request_over_the_wire_) {
- return;
- }
-
- std::shared_ptr<Request> req;
- switch (connected_) {
- case kNotYetConnected:
- return;
- case kConnecting:
- return;
- case kHandshaking:
- return;
- case kAuthenticating:
- if (auth_requests_.empty()) {
- return;
- }
- req = auth_requests_.front();
- auth_requests_.erase(auth_requests_.begin());
- break;
- case kConnected:
- if (pending_requests_.empty()) {
- return;
- }
- req = pending_requests_.front();
- pending_requests_.erase(pending_requests_.begin());
- break;
- case kDisconnected:
- LOG_DEBUG(kRPC, << "RpcConnectionImpl::FlushPendingRequests attempted to flush a " << ToString(connected_) << " connection");
- return;
- default:
- LOG_DEBUG(kRPC, << "RpcConnectionImpl::FlushPendingRequests invalid state: " << ToString(connected_));
- return;
- }
-
- std::shared_ptr<RpcConnection> shared_this = shared_from_this();
- auto weak_this = std::weak_ptr<RpcConnection>(shared_this);
- auto weak_req = std::weak_ptr<Request>(req);
-
- std::shared_ptr<std::string> payload = std::make_shared<std::string>();
- req->GetPacket(payload.get());
- if (!payload->empty()) {
- assert(requests_on_fly_.find(req->call_id()) == requests_on_fly_.end());
- requests_on_fly_[req->call_id()] = req;
- request_over_the_wire_ = req;
-
- req->timer().expires_from_now(
- std::chrono::milliseconds(options_.rpc_timeout));
- req->timer().async_wait([weak_this, weak_req, this](const ::asio::error_code &ec) {
- auto timeout_this = weak_this.lock();
- auto timeout_req = weak_req.lock();
- if (timeout_this && timeout_req)
- this->HandleRpcTimeout(timeout_req, ec);
- });
-
- asio::async_write(socket_, asio::buffer(*payload),
- [shared_this, this, payload](const ::asio::error_code &ec,
- size_t size) {
- OnSendCompleted(ec, size);
- });
- } else { // Nothing to send for this request, inform the handler immediately
- io_service().post(
- // Never hold locks when calling a callback
- [req]() { req->OnResponseArrived(nullptr, Status::OK()); }
- );
-
- // Reschedule to flush the next one
- AsyncFlushPendingRequests();
- }
-}
-
-
-template <class Socket>
-void RpcConnectionImpl<Socket>::OnRecvCompleted(const ::asio::error_code &original_ec,
- size_t) {
- using std::placeholders::_1;
- using std::placeholders::_2;
- std::lock_guard<std::mutex> state_lock(connection_state_lock_);
-
- ::asio::error_code my_ec(original_ec);
-
- LOG_TRACE(kRPC, << "RpcConnectionImpl::OnRecvCompleted called");
-
- std::shared_ptr<RpcConnection> shared_this = shared_from_this();
-
- if(event_handlers_) {
- event_response event_resp = event_handlers_->call(FS_NN_READ_EVENT, cluster_name_.c_str(), 0);
-#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
- if (event_resp.response() == event_response::kTest_Error) {
- my_ec = std::make_error_code(std::errc::network_down);
- }
-#endif
- }
-
- switch (my_ec.value()) {
- case 0:
- // No errors
- break;
- case asio::error::operation_aborted:
- // The event loop has been shut down. Ignore the error.
- return;
- default:
- LOG_WARN(kRPC, << "Network error during RPC read: " << my_ec.message());
- CommsError(ToStatus(my_ec));
- return;
- }
-
- if (!current_response_state_) { /* start a new one */
- current_response_state_ = std::make_shared<Response>();
- }
-
- if (current_response_state_->state_ == Response::kReadLength) {
- current_response_state_->state_ = Response::kReadContent;
- auto buf = ::asio::buffer(reinterpret_cast<char *>(¤t_response_state_->length_),
- sizeof(current_response_state_->length_));
- asio::async_read(
- socket_, buf,
- [shared_this, this](const ::asio::error_code &ec, size_t size) {
- OnRecvCompleted(ec, size);
- });
- } else if (current_response_state_->state_ == Response::kReadContent) {
- current_response_state_->state_ = Response::kParseResponse;
- current_response_state_->length_ = ntohl(current_response_state_->length_);
- current_response_state_->data_.resize(current_response_state_->length_);
- asio::async_read(
- socket_, ::asio::buffer(current_response_state_->data_),
- [shared_this, this](const ::asio::error_code &ec, size_t size) {
- OnRecvCompleted(ec, size);
- });
- } else if (current_response_state_->state_ == Response::kParseResponse) {
- // Check return status from the RPC response. We may have received a msg
- // indicating a server side error.
-
- Status stat = HandleRpcResponse(current_response_state_);
-
- if(stat.get_server_exception_type() == Status::kStandbyException) {
- // May need to bail out, connect to new NN, and restart loop
- LOG_INFO(kRPC, << "Communicating with standby NN, attempting to reconnect");
- }
-
- current_response_state_ = nullptr;
- StartReading();
- }
-}
-
-template <class Socket>
-void RpcConnectionImpl<Socket>::Disconnect() {
- assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
-
- LOG_INFO(kRPC, << "RpcConnectionImpl::Disconnect called");
-
- request_over_the_wire_.reset();
- if (connected_ == kConnecting || connected_ == kHandshaking || connected_ == kAuthenticating || connected_ == kConnected) {
- // Don't print out errors, we were expecting a disconnect here
- SafeDisconnect(get_asio_socket_ptr(&socket_));
- }
- connected_ = kDisconnected;
-}
-}
-
-#endif
+} // end namespace hdfs
+#endif // end include Guard
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e53da2d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
new file mode 100644
index 0000000..7accaf8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
@@ -0,0 +1,446 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "rpc_engine.h"
+#include "rpc_connection_impl.h"
+#include "sasl_protocol.h"
+
+#include "RpcHeader.pb.h"
+#include "ProtobufRpcEngine.pb.h"
+#include "IpcConnectionContext.pb.h"
+
+namespace hdfs {
+
+namespace pb = ::google::protobuf;
+namespace pbio = ::google::protobuf::io;
+
+using namespace ::hadoop::common;
+using namespace ::std::placeholders;
+
+static const int kNoRetry = -1;
+
+static void AddHeadersToPacket(
+ std::string *res, std::initializer_list<const pb::MessageLite *> headers,
+ const std::string *payload) {
+ int len = 0;
+ std::for_each(
+ headers.begin(), headers.end(),
+ [&len](const pb::MessageLite *v) { len += DelimitedPBMessageSize(v); });
+
+ if (payload) {
+ len += payload->size();
+ }
+
+ int net_len = htonl(len);
+ res->reserve(res->size() + sizeof(net_len) + len);
+
+ pbio::StringOutputStream ss(res);
+ pbio::CodedOutputStream os(&ss);
+ os.WriteRaw(reinterpret_cast<const char *>(&net_len), sizeof(net_len));
+
+ uint8_t *buf = os.GetDirectBufferForNBytesAndAdvance(len);
+ assert(buf);
+
+ std::for_each(
+ headers.begin(), headers.end(), [&buf](const pb::MessageLite *v) {
+ buf = pbio::CodedOutputStream::WriteVarint32ToArray(v->ByteSize(), buf);
+ buf = v->SerializeWithCachedSizesToArray(buf);
+ });
+
+ if (payload) {
+ buf = os.WriteStringToArray(*payload, buf);
+ }
+}
+
+RpcConnection::~RpcConnection() {}
+
+RpcConnection::RpcConnection(LockFreeRpcEngine *engine)
+ : engine_(engine),
+ connected_(kNotYetConnected) {}
+
+::asio::io_service &RpcConnection::io_service() {
+ return engine_->io_service();
+}
+
+void RpcConnection::StartReading() {
+ auto shared_this = shared_from_this();
+ io_service().post([shared_this, this] () {
+ OnRecvCompleted(::asio::error_code(), 0);
+ });
+}
+
+void RpcConnection::HandshakeComplete(const Status &s) {
+ std::lock_guard<std::mutex> state_lock(connection_state_lock_);
+
+ LOG_TRACE(kRPC, << "RpcConnectionImpl::HandshakeComplete called");
+
+ if (s.ok()) {
+ if (connected_ == kHandshaking) {
+ auto shared_this = shared_from_this();
+
+ connected_ = kAuthenticating;
+ if (auth_info_.useSASL()) {
+#ifdef USE_SASL
+ sasl_protocol_ = std::make_shared<SaslProtocol>(cluster_name_, auth_info_, shared_from_this());
+ sasl_protocol_->SetEventHandlers(event_handlers_);
+ sasl_protocol_->Authenticate([shared_this, this](
+ const Status & status, const AuthInfo & new_auth_info) {
+ AuthComplete(status, new_auth_info); } );
+#else
+ AuthComplete_locked(Status::Error("SASL is required, but no SASL library was found"), auth_info_);
+#endif
+ } else {
+ AuthComplete_locked(Status::OK(), auth_info_);
+ }
+ }
+ } else {
+ CommsError(s);
+ };
+}
+
+void RpcConnection::AuthComplete(const Status &s, const AuthInfo & new_auth_info) {
+ std::lock_guard<std::mutex> state_lock(connection_state_lock_);
+ AuthComplete_locked(s, new_auth_info);
+}
+
+void RpcConnection::AuthComplete_locked(const Status &s, const AuthInfo & new_auth_info) {
+ assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
+ LOG_TRACE(kRPC, << "RpcConnectionImpl::AuthComplete called");
+
+ // Free the sasl_protocol object
+ sasl_protocol_.reset();
+
+ if (s.ok()) {
+ auth_info_ = new_auth_info;
+
+ auto shared_this = shared_from_this();
+ SendContext([shared_this, this](const Status & s) {
+ ContextComplete(s);
+ });
+ } else {
+ CommsError(s);
+ };
+}
+
+void RpcConnection::ContextComplete(const Status &s) {
+ std::lock_guard<std::mutex> state_lock(connection_state_lock_);
+
+ LOG_TRACE(kRPC, << "RpcConnectionImpl::ContextComplete called");
+
+ if (s.ok()) {
+ if (connected_ == kAuthenticating) {
+ connected_ = kConnected;
+ }
+ FlushPendingRequests();
+ } else {
+ CommsError(s);
+ };
+}
+
+void RpcConnection::AsyncFlushPendingRequests() {
+ std::shared_ptr<RpcConnection> shared_this = shared_from_this();
+ io_service().post([shared_this, this]() {
+ std::lock_guard<std::mutex> state_lock(connection_state_lock_);
+
+ LOG_TRACE(kRPC, << "RpcConnection::AsyncFlushPendingRequests called (connected=" << ToString(connected_) << ")");
+
+ if (!request_over_the_wire_) {
+ FlushPendingRequests();
+ }
+ });
+}
+
+Status RpcConnection::HandleRpcResponse(std::shared_ptr<Response> response) {
+ assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
+
+ response->ar.reset(new pbio::ArrayInputStream(&response->data_[0], response->data_.size()));
+ response->in.reset(new pbio::CodedInputStream(response->ar.get()));
+ response->in->PushLimit(response->data_.size());
+ RpcResponseHeaderProto h;
+ ReadDelimitedPBMessage(response->in.get(), &h);
+
+ auto req = RemoveFromRunningQueue(h.callid());
+ if (!req) {
+ LOG_WARN(kRPC, << "RPC response with Unknown call id " << h.callid());
+ if((int32_t)h.callid() == RpcEngine::kCallIdSasl) {
+ return Status::AuthenticationFailed("You have an unsecured client connecting to a secured server");
+ } else {
+ return Status::Error("Rpc response with unknown call id");
+ }
+ }
+
+ Status status;
+ if(event_handlers_) {
+ event_response event_resp = event_handlers_->call(FS_NN_READ_EVENT, cluster_name_.c_str(), 0);
+#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
+ if (event_resp.response() == event_response::kTest_Error) {
+ status = event_resp.status();
+ }
+#endif
+ }
+
+ if (status.ok() && h.has_exceptionclassname()) {
+ status =
+ Status::Exception(h.exceptionclassname().c_str(), h.errormsg().c_str());
+ }
+
+ if(status.get_server_exception_type() == Status::kStandbyException) {
+ LOG_WARN(kRPC, << "Tried to connect to standby. status = " << status.ToString());
+
+ // We got the request back, but it needs to be resent to the other NN
+ std::vector<std::shared_ptr<Request>> reqs_to_redirect = {req};
+ PrependRequests_locked(reqs_to_redirect);
+
+ CommsError(status);
+ return status;
+ }
+
+ io_service().post([req, response, status]() {
+ req->OnResponseArrived(response->in.get(), status); // Never call back while holding a lock
+ });
+
+ return Status::OK();
+}
+
+void RpcConnection::HandleRpcTimeout(std::shared_ptr<Request> req,
+ const ::asio::error_code &ec) {
+ if (ec.value() == asio::error::operation_aborted) {
+ return;
+ }
+
+ std::lock_guard<std::mutex> state_lock(connection_state_lock_);
+ auto r = RemoveFromRunningQueue(req->call_id());
+ if (!r) {
+ // The RPC might have been finished and removed from the queue
+ return;
+ }
+
+ Status stat = ToStatus(ec ? ec : make_error_code(::asio::error::timed_out));
+
+ r->OnResponseArrived(nullptr, stat);
+}
+
+std::shared_ptr<std::string> RpcConnection::PrepareHandshakePacket() {
+ assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
+
+ /** From Client.java:
+ *
+ * Write the connection header - this is sent when connection is established
+ * +----------------------------------+
+ * | "hrpc" 4 bytes |
+ * +----------------------------------+
+ * | Version (1 byte) |
+ * +----------------------------------+
+ * | Service Class (1 byte) |
+ * +----------------------------------+
+ * | AuthProtocol (1 byte) |
+ * +----------------------------------+
+ *
+ * AuthProtocol: 0->none, -33->SASL
+ */
+
+ char auth_protocol = auth_info_.useSASL() ? -33 : 0;
+ const char handshake_header[] = {'h', 'r', 'p', 'c',
+ RpcEngine::kRpcVersion, 0, auth_protocol};
+ auto res =
+ std::make_shared<std::string>(handshake_header, sizeof(handshake_header));
+
+ return res;
+}
+
+std::shared_ptr<std::string> RpcConnection::PrepareContextPacket() {
+ // This needs to be send after the SASL handshake, and
+ // after the SASL handshake (if any)
+ assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
+
+ auto res = std::make_shared<std::string>();
+
+ RpcRequestHeaderProto h;
+ h.set_rpckind(RPC_PROTOCOL_BUFFER);
+ h.set_rpcop(RpcRequestHeaderProto::RPC_FINAL_PACKET);
+ h.set_callid(RpcEngine::kCallIdConnectionContext);
+ h.set_clientid(engine_->client_name());
+
+ IpcConnectionContextProto handshake;
+ handshake.set_protocol(engine_->protocol_name());
+ const std::string & user_name = auth_info_.getUser();
+ if (!user_name.empty()) {
+ *handshake.mutable_userinfo()->mutable_effectiveuser() = user_name;
+ }
+ AddHeadersToPacket(res.get(), {&h, &handshake}, nullptr);
+
+ return res;
+}
+
+void RpcConnection::AsyncRpc(
+ const std::string &method_name, const ::google::protobuf::MessageLite *req,
+ std::shared_ptr<::google::protobuf::MessageLite> resp,
+ const RpcCallback &handler) {
+ std::lock_guard<std::mutex> state_lock(connection_state_lock_);
+ AsyncRpc_locked(method_name, req, resp, handler);
+}
+
+void RpcConnection::AsyncRpc_locked(
+ const std::string &method_name, const ::google::protobuf::MessageLite *req,
+ std::shared_ptr<::google::protobuf::MessageLite> resp,
+ const RpcCallback &handler) {
+ assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
+
+ auto wrapped_handler =
+ [resp, handler](pbio::CodedInputStream *is, const Status &status) {
+ if (status.ok()) {
+ if (is) { // Connect messages will not have an is
+ ReadDelimitedPBMessage(is, resp.get());
+ }
+ }
+ handler(status);
+ };
+
+ int call_id = (method_name != SASL_METHOD_NAME ? engine_->NextCallId() : RpcEngine::kCallIdSasl);
+ auto r = std::make_shared<Request>(engine_, method_name, call_id, req,
+ std::move(wrapped_handler));
+ auto r_vector = std::vector<std::shared_ptr<Request> > (1, r);
+ SendRpcRequests(r_vector);
+}
+
+void RpcConnection::AsyncRpc(const std::vector<std::shared_ptr<Request> > & requests) {
+ std::lock_guard<std::mutex> state_lock(connection_state_lock_);
+ SendRpcRequests(requests);
+}
+
+void RpcConnection::SendRpcRequests(const std::vector<std::shared_ptr<Request> > & requests) {
+ LOG_TRACE(kRPC, << "RpcConnection::SendRpcRequests[] called; connected=" << ToString(connected_));
+ assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
+
+ if (connected_ == kDisconnected) {
+ // Oops. The connection failed _just_ before the engine got a chance
+ // to send it. Register it as a failure
+ Status status = Status::ResourceUnavailable("RpcConnection closed before send.");
+ engine_->AsyncRpcCommsError(status, shared_from_this(), requests);
+ } else {
+ for (auto r: requests) {
+ if (r->method_name() != SASL_METHOD_NAME)
+ pending_requests_.push_back(r);
+ else
+ auth_requests_.push_back(r);
+ }
+ if (connected_ == kConnected || connected_ == kHandshaking || connected_ == kAuthenticating) { // Dont flush if we're waiting or handshaking
+ FlushPendingRequests();
+ }
+ }
+}
+
+
+void RpcConnection::PreEnqueueRequests(
+ std::vector<std::shared_ptr<Request>> requests) {
+ // Public method - acquire lock
+ std::lock_guard<std::mutex> state_lock(connection_state_lock_);
+
+ LOG_DEBUG(kRPC, << "RpcConnection::PreEnqueueRequests called");
+
+ assert(connected_ == kNotYetConnected);
+
+ pending_requests_.insert(pending_requests_.end(), requests.begin(),
+ requests.end());
+ // Don't start sending yet; will flush when connected
+}
+
+// Only call when already holding conn state lock
+void RpcConnection::PrependRequests_locked( std::vector<std::shared_ptr<Request>> requests) {
+ LOG_DEBUG(kRPC, << "RpcConnection::PrependRequests called");
+
+ pending_requests_.insert(pending_requests_.begin(), requests.begin(),
+ requests.end());
+ // Don't start sending yet; will flush when connected
+}
+
+void RpcConnection::SetEventHandlers(std::shared_ptr<LibhdfsEvents> event_handlers) {
+ std::lock_guard<std::mutex> state_lock(connection_state_lock_);
+ event_handlers_ = event_handlers;
+ if (sasl_protocol_) {
+ sasl_protocol_->SetEventHandlers(event_handlers);
+ }
+}
+
+void RpcConnection::SetClusterName(std::string cluster_name) {
+ std::lock_guard<std::mutex> state_lock(connection_state_lock_);
+ cluster_name_ = cluster_name;
+}
+
+void RpcConnection::CommsError(const Status &status) {
+ assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
+ LOG_DEBUG(kRPC, << "RpcConnection::CommsError called");
+
+ Disconnect();
+
+ // Anything that has been queued to the connection (on the fly or pending)
+ // will get dinged for a retry
+ std::vector<std::shared_ptr<Request>> requestsToReturn;
+ std::transform(requests_on_fly_.begin(), requests_on_fly_.end(),
+ std::back_inserter(requestsToReturn),
+ std::bind(&RequestOnFlyMap::value_type::second, _1));
+ requests_on_fly_.clear();
+
+ requestsToReturn.insert(requestsToReturn.end(),
+ std::make_move_iterator(pending_requests_.begin()),
+ std::make_move_iterator(pending_requests_.end()));
+ pending_requests_.clear();
+
+ engine_->AsyncRpcCommsError(status, shared_from_this(), requestsToReturn);
+}
+
+void RpcConnection::ClearAndDisconnect(const ::asio::error_code &ec) {
+ Disconnect();
+ std::vector<std::shared_ptr<Request>> requests;
+ std::transform(requests_on_fly_.begin(), requests_on_fly_.end(),
+ std::back_inserter(requests),
+ std::bind(&RequestOnFlyMap::value_type::second, _1));
+ requests_on_fly_.clear();
+ requests.insert(requests.end(),
+ std::make_move_iterator(pending_requests_.begin()),
+ std::make_move_iterator(pending_requests_.end()));
+ pending_requests_.clear();
+ for (const auto &req : requests) {
+ req->OnResponseArrived(nullptr, ToStatus(ec));
+ }
+}
+
+std::shared_ptr<Request> RpcConnection::RemoveFromRunningQueue(int call_id) {
+ assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
+ auto it = requests_on_fly_.find(call_id);
+ if (it == requests_on_fly_.end()) {
+ return std::shared_ptr<Request>();
+ }
+
+ auto req = it->second;
+ requests_on_fly_.erase(it);
+ return req;
+}
+
+std::string RpcConnection::ToString(ConnectedState connected) {
+ switch(connected) {
+ case kNotYetConnected: return "NotYetConnected";
+ case kConnecting: return "Connecting";
+ case kHandshaking: return "Handshaking";
+ case kAuthenticating: return "Authenticating";
+ case kConnected: return "Connected";
+ case kDisconnected: return "Disconnected";
+ default: return "Invalid ConnectedState";
+ }
+}
+
+}// end namespace hdfs
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/52] [abbrv] hadoop git commit: HDFS-10740: libhdfs++: Implement
recursive directory generator. Contributed by Anatoli Shein.
Posted by jh...@apache.org.
HDFS-10740: libhdfs++: Implement recursive directory generator. Contributed by Anatoli Shein.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cbb3e9fe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cbb3e9fe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cbb3e9fe
Branch: refs/heads/HDFS-8707
Commit: cbb3e9fe58d1d472cdfce00e4471d26b00cb9b83
Parents: 2a8edd4
Author: James <jh...@apache.org>
Authored: Fri Aug 12 11:44:40 2016 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../libhdfspp/examples/cpp/CMakeLists.txt | 1 +
.../examples/cpp/gendirs/CMakeLists.txt | 35 ++++
.../libhdfspp/examples/cpp/gendirs/gendirs.cpp | 166 +++++++++++++++++++
3 files changed, 202 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cbb3e9fe/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/CMakeLists.txt
index 76880cd..183299a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/CMakeLists.txt
@@ -17,3 +17,4 @@
#
add_subdirectory(cat)
+add_subdirectory(gendirs)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cbb3e9fe/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/gendirs/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/gendirs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/gendirs/CMakeLists.txt
new file mode 100644
index 0000000..90cddf0
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/gendirs/CMakeLists.txt
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Default LIBHDFSPP_DIR to the default install location. You can override
+# it by add -DLIBHDFSPP_DIR=... to your cmake invocation
+set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX})
+
+include_directories( ${LIBHDFSPP_DIR}/include )
+link_directories( ${LIBHDFSPP_DIR}/lib )
+
+add_executable(gendirs_cpp gendirs.cpp)
+target_link_libraries(gendirs_cpp hdfspp)
+
+# Several examples in different languages need to produce executables with
+# same names. To allow executables with same names we keep their CMake
+# names different, but specify their executable names as follows:
+set_target_properties( gendirs_cpp
+ PROPERTIES
+ OUTPUT_NAME "gendirs"
+)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cbb3e9fe/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/gendirs/gendirs.cpp
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/gendirs/gendirs.cpp b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/gendirs/gendirs.cpp
new file mode 100644
index 0000000..9f5ae5a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/gendirs/gendirs.cpp
@@ -0,0 +1,166 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+/**
+ * A recursive directory generator tool.
+ *
+ * Generates a directory tree with specified depth and fanout starting from
+ * a given path. Generation is asynchronous.
+ *
+ * Usage: gendirs [hdfs://[<hostname>:<port>]]/<path-to-dir> <depth> <fanout>
+ *
+ * Example: gendirs hdfs://localhost.localdomain:9433/dir0 3 10
+ *
+ * @param path-to-dir Absolute path to the directory tree root where the
+ * directory tree will be generated
+ * @param depth Depth of the directory tree (number of levels from
+ * root to leaves)
+ * @param fanout Fanout of each directory (number of sub-directories to
+ * be created inside each directory except leaf directories)
+ *
+ **/
+
+#include "hdfspp/hdfspp.h"
+#include "fs/namenode_operations.h"
+#include "common/hdfs_configuration.h"
+#include "common/configuration_loader.h"
+#include "common/uri.h"
+
+#include <google/protobuf/io/coded_stream.h>
+#include <future>
+
+using namespace std;
+using namespace hdfs;
+
+#define SCHEME "hdfs"
+
+void GenerateDirectories (shared_ptr<FileSystem> fs, int depth, int level, int fanout, string path, vector<future<Status>> & futures) {
+ //Level contains our current depth in the directory tree
+ if(level < depth) {
+ for(int i = 0; i < fanout; i++){
+ //Recursive calls to cover all possible paths from the root to the leave nodes
+ GenerateDirectories(fs, depth, level+1, fanout, path + "dir" + to_string(i) + "/", futures);
+ }
+ } else {
+ //We have reached the leaf nodes and now start making calls to create directories
+ //We make a promise which will be set when the call finishes and executes our handler
+ auto callstate = make_shared<promise<Status>>();
+ //Extract a future from this promise
+ future<Status> future(callstate->get_future());
+ //Save this future to the vector of futures which will be used to wait on all promises
+ //after the whole recursion is done
+ futures.push_back(move(future));
+ //Create a handler that will be executed when Mkdirs is done
+ auto handler = [callstate](const Status &s) {
+ callstate->set_value(s);
+ };
+ //Asynchronous call to create this directory along with all missing parent directories
+ fs->Mkdirs(path, NameNodeOperations::GetDefaultPermissionMask(), true, handler);
+ }
+}
+
+int main(int argc, char *argv[]) {
+ if (argc != 4) {
+ cerr << "usage: gendirs [hdfs://[<hostname>:<port>]]/<path-to-dir> <depth> <fanout>" << endl;
+ return 1;
+ }
+
+ optional<URI> uri;
+ const string uri_path = argv[1];
+ const int depth = stoi(argv[2]);
+ const int fanout = stoi(argv[3]);
+
+ //Separate check for scheme is required, otherwise common/uri.h library causes memory issues under valgrind
+ size_t scheme_end = uri_path.find("://");
+ if (scheme_end != string::npos) {
+ if(uri_path.substr(0, string(SCHEME).size()).compare(SCHEME) != 0) {
+ cerr << "Scheme " << uri_path.substr(0, scheme_end) << ":// is not supported" << endl;
+ return 1;
+ } else {
+ uri = URI::parse_from_string(uri_path);
+ }
+ }
+ if (!uri) {
+ cerr << "Malformed URI: " << uri_path << endl;
+ return 1;
+ }
+
+ ConfigurationLoader loader;
+ optional<HdfsConfiguration> config = loader.LoadDefaultResources<HdfsConfiguration>();
+ const char * envHadoopConfDir = getenv("HADOOP_CONF_DIR");
+ if (envHadoopConfDir && (*envHadoopConfDir != 0) ) {
+ config = loader.OverlayResourceFile(*config, string(envHadoopConfDir) + "/core-site.xml");
+ }
+
+ Options options;
+ options.rpc_timeout = numeric_limits<int>::max();
+ if(config){
+ options = config->GetOptions();
+ }
+
+ IoService * io_service = IoService::New();
+
+ FileSystem *fs_raw = FileSystem::New(io_service, "", options);
+ if (!fs_raw) {
+ cerr << "Could not create FileSystem object" << endl;
+ return 1;
+ }
+ //Wrapping fs_raw into a unique pointer to guarantee deletion
+ shared_ptr<FileSystem> fs(fs_raw);
+
+ //Get port from the uri, otherwise use the default port
+ string port = to_string(uri->get_port().value_or(8020));
+ Status stat = fs->Connect(uri->get_host(), port);
+ if (!stat.ok()) {
+ cerr << "Could not connect to " << uri->get_host() << ":" << port << endl;
+ return 1;
+ }
+
+ /**
+ * We do not want the recursion to block on anything, therefore we will be
+ * making asynchronous calls recursively, and then just waiting for all
+ * the calls to finish.
+ *
+ * This array of futures will be populated by the recursive function below.
+ * Each new asynchronous Mkdirs call will add a future to this vector, and will
+ * create a promise, which will only be set when the call was completed and
+ * processed. After the whole recursion is complete we will need to wait until
+ * all promises are set before we can exit.
+ **/
+ vector<future<Status>> futures;
+
+ GenerateDirectories(fs, depth, 0, fanout, uri->get_path() + "/", futures);
+
+ /**
+ * We are waiting here until all promises are set, and checking whether
+ * the returned statuses contained any errors.
+ **/
+ for(future<Status> &fs : futures){
+ Status stat = fs.get();
+ if (!stat.ok()) {
+ cerr << "Error: " << stat.ToString() << endl;
+ }
+ }
+
+ cout << "All done!" << endl;
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/52] [abbrv] hadoop git commit: HDFS-10595: libhdfs++: deconflate
client name and client id
Posted by jh...@apache.org.
HDFS-10595: libhdfs++: deconflate client name and client id
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/69cb05d2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/69cb05d2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/69cb05d2
Branch: refs/heads/HDFS-8707
Commit: 69cb05d2920a02435921028b70ed88c587823c05
Parents: d22e4b2
Author: Bob Hansen <bo...@hpe.com>
Authored: Fri Sep 30 11:42:51 2016 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../main/native/libhdfspp/lib/common/util.cc | 34 +++++++++-----------
.../native/libhdfspp/lib/rpc/rpc_connection.cc | 2 +-
.../main/native/libhdfspp/lib/rpc/rpc_engine.cc | 23 +++++++++++++
.../main/native/libhdfspp/lib/rpc/rpc_engine.h | 5 +++
4 files changed, 44 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69cb05d2/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
index 853ada8..ede6acd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
@@ -21,6 +21,10 @@
#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
#include <exception>
+#include <sstream>
+#include <iostream>
+#include <iomanip>
+#include <thread>
namespace hdfs {
@@ -56,25 +60,17 @@ std::string SerializeDelimitedProtobufMessage(const ::google::protobuf::MessageL
std::string GetRandomClientName() {
- /**
- * The server is requesting a 16-byte UUID:
- * https://github.com/c9n/hadoop/blob/master/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientId.java
- *
- * This function generates a 16-byte UUID (version 4):
- * https://en.wikipedia.org/wiki/Universally_unique_identifier#Version_4_.28random.29
- **/
- unsigned char buf[16];
- RAND_pseudo_bytes(buf, sizeof(buf));
-
- //clear the first four bits of byte 6 then set the second bit
- buf[6] = (buf[6] & 0x0f) | 0x40;
-
- //clear the second bit of byte 8 and set the first bit
- buf[8] = (buf[8] & 0xbf) | 0x80;
-
- std::stringstream ss;
- ss << std::string(reinterpret_cast<char *>(buf), sizeof(buf));
- return ss.str();
+ std::vector<unsigned char>buf(8);
+ RAND_pseudo_bytes(&buf[0], 8);
+
+ std::ostringstream oss;
+ oss << "DFSClient_" << getpid() << "_" <<
+ std::this_thread::get_id() << "_" <<
+ std::setw(2) << std::hex << std::uppercase << std::setfill('0');
+ for (unsigned char b: buf)
+ oss << static_cast<unsigned>(b);
+
+ return oss.str();
}
std::string SafeDisconnect(asio::ip::tcp::socket *sock) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69cb05d2/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
index b6136c2..2068614 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
@@ -102,7 +102,7 @@ static void SetRequestHeader(LockFreeRpcEngine *engine, int call_id,
rpc_header->set_callid(call_id);
if (retry_count != kNoRetry)
rpc_header->set_retrycount(retry_count);
- rpc_header->set_clientid(engine->client_name());
+ rpc_header->set_clientid(engine->client_id());
req_header->set_methodname(method_name);
req_header->set_declaringclassprotocolname(engine->protocol_name());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69cb05d2/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
index 89d6a67..4de6704 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
@@ -145,6 +145,7 @@ RpcEngine::RpcEngine(::asio::io_service *io_service, const Options &options,
: io_service_(io_service),
options_(options),
client_name_(client_name),
+ client_id_(getRandomClientId()),
protocol_name_(protocol_name),
protocol_version_(protocol_version),
call_id_(0),
@@ -206,6 +207,28 @@ std::unique_ptr<const RetryPolicy> RpcEngine::MakeRetryPolicy(const Options &opt
}
}
+std::string RpcEngine::getRandomClientId()
+{
+ /**
+ * The server is requesting a 16-byte UUID:
+ * https://github.com/c9n/hadoop/blob/master/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientId.java
+ *
+ * This function generates a 16-byte UUID (version 4):
+ * https://en.wikipedia.org/wiki/Universally_unique_identifier#Version_4_.28random.29
+ **/
+ std::vector<unsigned char>buf(16);
+ RAND_pseudo_bytes(&buf[0], buf.size());
+
+ //clear the first four bits of byte 6 then set the second bit
+ buf[6] = (buf[6] & 0x0f) | 0x40;
+
+ //clear the second bit of byte 8 and set the first bit
+ buf[8] = (buf[8] & 0xbf) | 0x80;
+ return std::string(reinterpret_cast<const char*>(&buf[0]), buf.size());
+}
+
+
+
void RpcEngine::TEST_SetRpcConnection(std::shared_ptr<RpcConnection> conn) {
conn_ = conn;
retry_policy_ = std::move(MakeRetryPolicy(options_));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69cb05d2/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
index efaf407..9191ab2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
@@ -269,6 +269,7 @@ public:
virtual int NextCallId() = 0;
virtual const std::string &client_name() const = 0;
+ virtual const std::string &client_id() const = 0;
virtual const std::string &user_name() const = 0;
virtual const std::string &protocol_name() const = 0;
virtual int protocol_version() const = 0;
@@ -380,6 +381,7 @@ class RpcEngine : public LockFreeRpcEngine {
std::unique_ptr<const RetryPolicy> TEST_GenerateRetryPolicyUsingOptions();
const std::string &client_name() const override { return client_name_; }
+ const std::string &client_id() const override { return client_id_; }
const std::string &user_name() const override { return auth_info_.getUser(); }
const std::string &protocol_name() const override { return protocol_name_; }
int protocol_version() const override { return protocol_version_; }
@@ -394,6 +396,8 @@ protected:
virtual std::shared_ptr<RpcConnection> NewConnection();
virtual std::unique_ptr<const RetryPolicy> MakeRetryPolicy(const Options &options);
+ static std::string getRandomClientId();
+
// Remember all of the last endpoints in case we need to reconnect and retry
std::vector<::asio::ip::tcp::endpoint> last_endpoints_;
@@ -401,6 +405,7 @@ private:
::asio::io_service * const io_service_;
const Options options_;
const std::string client_name_;
+ const std::string client_id_;
const std::string protocol_name_;
const int protocol_version_;
std::unique_ptr<const RetryPolicy> retry_policy_; //null --> no retry
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/52] [abbrv] hadoop git commit: HDFS-11294: libhdfs++: Segfault in
HA failover if DNS lookup for both Namenodes fails. Contributed by James
Clampffer.
Posted by jh...@apache.org.
HDFS-11294: libhdfs++: Segfault in HA failover if DNS lookup for both Namenodes fails. Contributed by James Clampffer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/48db24a4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/48db24a4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/48db24a4
Branch: refs/heads/HDFS-8707
Commit: 48db24a430f5911a564d08f9ae927aaa9d81cd50
Parents: 12942f6
Author: James <jh...@apache.org>
Authored: Mon Oct 30 12:07:02 2017 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../native/libhdfspp/include/hdfspp/events.h | 14 +-
.../native/libhdfspp/lib/bindings/c/hdfs.cc | 90 +++++++++++-
.../libhdfspp/lib/rpc/namenode_tracker.cc | 66 +++++----
.../native/libhdfspp/lib/rpc/namenode_tracker.h | 9 +-
.../main/native/libhdfspp/lib/rpc/rpc_engine.cc | 32 ++++-
.../main/native/libhdfspp/tests/CMakeLists.txt | 12 +-
.../libhdfspp/tests/hdfs_config_connect_bugs.cc | 136 +++++++++++++++++++
.../native/libhdfspp/tests/rpc_engine_test.cc | 16 ++-
8 files changed, 325 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/48db24a4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/events.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/events.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/events.h
index 496703a..83c0deb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/events.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/events.h
@@ -37,17 +37,23 @@ static constexpr const char * FS_NN_CONNECT_EVENT = "NN::connect";
static constexpr const char * FS_NN_READ_EVENT = "NN::read";
static constexpr const char * FS_NN_WRITE_EVENT = "NN::write";
+static constexpr const char * FILE_DN_CONNECT_EVENT = "DN::connect";
+static constexpr const char * FILE_DN_READ_EVENT = "DN::read";
+static constexpr const char * FILE_DN_WRITE_EVENT = "DN::write";
+
+
// NN failover event due to issues with the current NN; might be standby, might be dead.
// Invokes the fs_event_callback using the nameservice name in the cluster string.
// The uint64_t value argument holds an address that can be reinterpreted as a const char *
// and provides the full URI of the node the failover will attempt to connect to next.
static constexpr const char * FS_NN_FAILOVER_EVENT = "NN::failover";
-static constexpr const char * FILE_DN_CONNECT_EVENT = "DN::connect";
-static constexpr const char * FILE_DN_READ_EVENT = "DN::read";
-static constexpr const char * FILE_DN_WRITE_EVENT = "DN::write";
-
+// Invoked when RpcConnection tries to use an empty set of endpoints to figure out
+// which NN in a HA cluster to connect to.
+static constexpr const char * FS_NN_EMPTY_ENDPOINTS_EVENT = "NN::bad_failover::no_endpoints";
+// Invoked prior to determining if failed NN rpc calls should be retried or discarded.
+static constexpr const char * FS_NN_PRE_RPC_RETRY_EVENT = "NN::rpc::get_retry_action";
class event_response {
public:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/48db24a4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
index 3e9ee48..9a7c8b4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
@@ -41,6 +41,12 @@ using namespace std::placeholders;
static constexpr tPort kDefaultPort = 8020;
+/** Annotate what parts of the code below are implementatons of API functions
+ * and if they are normal vs. extended API.
+ */
+#define LIBHDFS_C_API
+#define LIBHDFSPP_EXT_API
+
/* Separate the handles used by the C api from the C++ API*/
struct hdfs_internal {
hdfs_internal(FileSystem *p) : filesystem_(p), working_directory_("/") {}
@@ -79,6 +85,7 @@ struct hdfsFile_internal {
thread_local std::string errstr;
/* Fetch last error that happened in this thread */
+LIBHDFSPP_EXT_API
int hdfsGetLastError(char *buf, int len) {
//No error message
if(errstr.empty()){
@@ -255,6 +262,7 @@ optional<std::string> getAbsolutePath(hdfsFS fs, const char* path) {
* C API implementations
**/
+LIBHDFS_C_API
int hdfsFileIsOpenForRead(hdfsFile file) {
/* files can only be open for reads at the moment, do a quick check */
if (!CheckHandle(file)){
@@ -263,6 +271,7 @@ int hdfsFileIsOpenForRead(hdfsFile file) {
return 1; // Update implementation when we get file writing
}
+LIBHDFS_C_API
int hdfsFileIsOpenForWrite(hdfsFile file) {
/* files can only be open for reads at the moment, so return false */
CheckHandle(file);
@@ -332,6 +341,7 @@ hdfsFS doHdfsConnect(optional<std::string> nn, optional<tPort> port, optional<st
}
}
+LIBHDFSPP_EXT_API
hdfsFS hdfsAllocateFileSystem(struct hdfsBuilder *bld) {
// Same idea as the first half of doHdfsConnect, but return the wrapped FS before
// connecting.
@@ -367,6 +377,7 @@ hdfsFS hdfsAllocateFileSystem(struct hdfsBuilder *bld) {
return nullptr;
}
+LIBHDFSPP_EXT_API
int hdfsConnectAllocated(hdfsFS fs, struct hdfsBuilder *bld) {
if(!CheckSystem(fs)) {
return ENODEV;
@@ -420,24 +431,29 @@ int hdfsConnectAllocated(hdfsFS fs, struct hdfsBuilder *bld) {
return 0;
}
+LIBHDFS_C_API
hdfsFS hdfsConnect(const char *nn, tPort port) {
return hdfsConnectAsUser(nn, port, "");
}
+LIBHDFS_C_API
hdfsFS hdfsConnectAsUser(const char* nn, tPort port, const char *user) {
return doHdfsConnect(std::string(nn), port, std::string(user), Options());
}
+LIBHDFS_C_API
hdfsFS hdfsConnectAsUserNewInstance(const char* nn, tPort port, const char *user ) {
//libhdfspp always returns a new instance
return doHdfsConnect(std::string(nn), port, std::string(user), Options());
}
+LIBHDFS_C_API
hdfsFS hdfsConnectNewInstance(const char* nn, tPort port) {
//libhdfspp always returns a new instance
return hdfsConnectAsUser(nn, port, "");
}
+LIBHDFSPP_EXT_API
int hdfsCancelPendingConnection(hdfsFS fs) {
// todo: stick an enum in hdfs_internal to check the connect state
if(!CheckSystem(fs)) {
@@ -458,6 +474,7 @@ int hdfsCancelPendingConnection(hdfsFS fs) {
}
}
+LIBHDFS_C_API
int hdfsDisconnect(hdfsFS fs) {
try
{
@@ -476,6 +493,7 @@ int hdfsDisconnect(hdfsFS fs) {
}
}
+LIBHDFS_C_API
hdfsFile hdfsOpenFile(hdfsFS fs, const char *path, int flags, int bufferSize,
short replication, tSize blocksize) {
try
@@ -512,6 +530,7 @@ hdfsFile hdfsOpenFile(hdfsFS fs, const char *path, int flags, int bufferSize,
}
}
+LIBHDFS_C_API
int hdfsCloseFile(hdfsFS fs, hdfsFile file) {
try
{
@@ -528,6 +547,7 @@ int hdfsCloseFile(hdfsFS fs, hdfsFile file) {
}
}
+LIBHDFS_C_API
char* hdfsGetWorkingDirectory(hdfsFS fs, char *buffer, size_t bufferSize) {
try
{
@@ -556,6 +576,7 @@ char* hdfsGetWorkingDirectory(hdfsFS fs, char *buffer, size_t bufferSize) {
}
}
+LIBHDFS_C_API
int hdfsSetWorkingDirectory(hdfsFS fs, const char* path) {
try
{
@@ -582,6 +603,7 @@ int hdfsSetWorkingDirectory(hdfsFS fs, const char* path) {
}
}
+LIBHDFS_C_API
int hdfsAvailable(hdfsFS fs, hdfsFile file) {
//Since we do not have read ahead implemented, return 0 if fs and file are good;
errno = 0;
@@ -591,6 +613,7 @@ int hdfsAvailable(hdfsFS fs, hdfsFile file) {
return 0;
}
+LIBHDFS_C_API
tOffset hdfsGetDefaultBlockSize(hdfsFS fs) {
try {
errno = 0;
@@ -604,6 +627,7 @@ tOffset hdfsGetDefaultBlockSize(hdfsFS fs) {
}
}
+LIBHDFS_C_API
tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path) {
try {
errno = 0;
@@ -633,6 +657,7 @@ tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path) {
}
}
+LIBHDFS_C_API
int hdfsSetReplication(hdfsFS fs, const char* path, int16_t replication) {
try {
errno = 0;
@@ -659,6 +684,7 @@ int hdfsSetReplication(hdfsFS fs, const char* path, int16_t replication) {
}
}
+LIBHDFS_C_API
int hdfsUtime(hdfsFS fs, const char* path, tTime mtime, tTime atime) {
try {
errno = 0;
@@ -682,6 +708,7 @@ int hdfsUtime(hdfsFS fs, const char* path, tTime mtime, tTime atime) {
}
}
+LIBHDFS_C_API
tOffset hdfsGetCapacity(hdfsFS fs) {
try {
errno = 0;
@@ -705,6 +732,7 @@ tOffset hdfsGetCapacity(hdfsFS fs) {
}
}
+LIBHDFS_C_API
tOffset hdfsGetUsed(hdfsFS fs) {
try {
errno = 0;
@@ -777,6 +805,7 @@ void StatInfoToHdfsFileInfo(hdfsFileInfo * file_info,
file_info->mLastAccess = stat_info.access_time;
}
+LIBHDFS_C_API
int hdfsExists(hdfsFS fs, const char *path) {
try {
errno = 0;
@@ -800,6 +829,7 @@ int hdfsExists(hdfsFS fs, const char *path) {
}
}
+LIBHDFS_C_API
hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char* path) {
try {
errno = 0;
@@ -828,6 +858,7 @@ hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char* path) {
}
}
+LIBHDFS_C_API
hdfsFileInfo *hdfsListDirectory(hdfsFS fs, const char* path, int *numEntries) {
try {
errno = 0;
@@ -868,6 +899,7 @@ hdfsFileInfo *hdfsListDirectory(hdfsFS fs, const char* path, int *numEntries) {
}
}
+LIBHDFS_C_API
void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries)
{
errno = 0;
@@ -880,6 +912,7 @@ void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries)
delete[] hdfsFileInfo;
}
+LIBHDFS_C_API
int hdfsCreateDirectory(hdfsFS fs, const char* path) {
try {
errno = 0;
@@ -904,6 +937,7 @@ int hdfsCreateDirectory(hdfsFS fs, const char* path) {
}
}
+LIBHDFS_C_API
int hdfsDelete(hdfsFS fs, const char* path, int recursive) {
try {
errno = 0;
@@ -927,6 +961,7 @@ int hdfsDelete(hdfsFS fs, const char* path, int recursive) {
}
}
+LIBHDFS_C_API
int hdfsRename(hdfsFS fs, const char* oldPath, const char* newPath) {
try {
errno = 0;
@@ -951,6 +986,7 @@ int hdfsRename(hdfsFS fs, const char* oldPath, const char* newPath) {
}
}
+LIBHDFS_C_API
int hdfsChmod(hdfsFS fs, const char* path, short mode){
try {
errno = 0;
@@ -977,6 +1013,7 @@ int hdfsChmod(hdfsFS fs, const char* path, short mode){
}
}
+LIBHDFS_C_API
int hdfsChown(hdfsFS fs, const char* path, const char *owner, const char *group){
try {
errno = 0;
@@ -1003,6 +1040,7 @@ int hdfsChown(hdfsFS fs, const char* path, const char *owner, const char *group)
}
}
+LIBHDFSPP_EXT_API
hdfsFileInfo * hdfsFind(hdfsFS fs, const char* path, const char* name, uint32_t * numEntries){
try {
errno = 0;
@@ -1041,6 +1079,7 @@ hdfsFileInfo * hdfsFind(hdfsFS fs, const char* path, const char* name, uint32_t
}
}
+LIBHDFSPP_EXT_API
int hdfsCreateSnapshot(hdfsFS fs, const char* path, const char* name) {
try {
errno = 0;
@@ -1068,6 +1107,7 @@ int hdfsCreateSnapshot(hdfsFS fs, const char* path, const char* name) {
}
}
+LIBHDFSPP_EXT_API
int hdfsDeleteSnapshot(hdfsFS fs, const char* path, const char* name) {
try {
errno = 0;
@@ -1125,6 +1165,7 @@ int hdfsRenameSnapshot(hdfsFS fs, const char* path, const char* old_name, const
}
+LIBHDFSPP_EXT_API
int hdfsAllowSnapshot(hdfsFS fs, const char* path) {
try {
errno = 0;
@@ -1148,6 +1189,7 @@ int hdfsAllowSnapshot(hdfsFS fs, const char* path) {
}
}
+LIBHDFSPP_EXT_API
int hdfsDisallowSnapshot(hdfsFS fs, const char* path) {
try {
errno = 0;
@@ -1171,6 +1213,7 @@ int hdfsDisallowSnapshot(hdfsFS fs, const char* path) {
}
}
+LIBHDFS_C_API
tSize hdfsPread(hdfsFS fs, hdfsFile file, tOffset position, void *buffer,
tSize length) {
try
@@ -1193,6 +1236,7 @@ tSize hdfsPread(hdfsFS fs, hdfsFile file, tOffset position, void *buffer,
}
}
+LIBHDFS_C_API
tSize hdfsRead(hdfsFS fs, hdfsFile file, void *buffer, tSize length) {
try
{
@@ -1215,12 +1259,14 @@ tSize hdfsRead(hdfsFS fs, hdfsFile file, void *buffer, tSize length) {
}
}
+LIBHDFS_C_API
int hdfsUnbufferFile(hdfsFile file) {
//Currently we are not doing any buffering
CheckHandle(file);
return -1;
}
+LIBHDFS_C_API
int hdfsFileGetReadStatistics(hdfsFile file, struct hdfsReadStatistics **stats) {
try
{
@@ -1239,6 +1285,7 @@ int hdfsFileGetReadStatistics(hdfsFile file, struct hdfsReadStatistics **stats)
}
}
+LIBHDFS_C_API
int hdfsFileClearReadStatistics(hdfsFile file) {
try
{
@@ -1255,16 +1302,19 @@ int hdfsFileClearReadStatistics(hdfsFile file) {
}
}
+LIBHDFS_C_API
int64_t hdfsReadStatisticsGetRemoteBytesRead(const struct hdfsReadStatistics *stats) {
return stats->totalBytesRead - stats->totalLocalBytesRead;
}
+LIBHDFS_C_API
void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats) {
errno = 0;
delete stats;
}
/* 0 on success, -1 on error*/
+LIBHDFS_C_API
int hdfsSeek(hdfsFS fs, hdfsFile file, tOffset desiredPos) {
try
{
@@ -1287,6 +1337,7 @@ int hdfsSeek(hdfsFS fs, hdfsFile file, tOffset desiredPos) {
}
}
+LIBHDFS_C_API
tOffset hdfsTell(hdfsFS fs, hdfsFile file) {
try
{
@@ -1326,7 +1377,7 @@ int hdfsCancel(hdfsFS fs, hdfsFile file) {
}
}
-
+LIBHDFSPP_EXT_API
int hdfsGetBlockLocations(hdfsFS fs, const char *path, struct hdfsBlockLocations ** locations_out)
{
try
@@ -1402,6 +1453,7 @@ int hdfsGetBlockLocations(hdfsFS fs, const char *path, struct hdfsBlockLocations
}
}
+LIBHDFSPP_EXT_API
int hdfsFreeBlockLocations(struct hdfsBlockLocations * blockLocations) {
errno = 0;
if (blockLocations == nullptr)
@@ -1422,6 +1474,7 @@ int hdfsFreeBlockLocations(struct hdfsBlockLocations * blockLocations) {
return 0;
}
+LIBHDFS_C_API
char*** hdfsGetHosts(hdfsFS fs, const char* path, tOffset start, tOffset length) {
try
{
@@ -1462,6 +1515,7 @@ char*** hdfsGetHosts(hdfsFS fs, const char* path, tOffset start, tOffset length)
}
}
+LIBHDFS_C_API
void hdfsFreeHosts(char ***blockHosts) {
errno = 0;
if (blockHosts == nullptr)
@@ -1526,6 +1580,7 @@ event_response file_callback_glue(libhdfspp_file_event_callback handler,
return event_response::make_ok();
}
+LIBHDFSPP_EXT_API
int hdfsPreAttachFSMonitor(libhdfspp_fs_event_callback handler, int64_t cookie)
{
fs_event_callback callback = std::bind(fs_callback_glue, handler, cookie, _1, _2, _3);
@@ -1533,7 +1588,7 @@ int hdfsPreAttachFSMonitor(libhdfspp_fs_event_callback handler, int64_t cookie)
return 0;
}
-
+LIBHDFSPP_EXT_API
int hdfsPreAttachFileMonitor(libhdfspp_file_event_callback handler, int64_t cookie)
{
file_event_callback callback = std::bind(file_callback_glue, handler, cookie, _1, _2, _3, _4);
@@ -1572,6 +1627,7 @@ hdfsBuilder::hdfsBuilder(const char * directory) :
config = LoadDefault(loader);
}
+LIBHDFS_C_API
struct hdfsBuilder *hdfsNewBuilder(void)
{
try
@@ -1587,18 +1643,21 @@ struct hdfsBuilder *hdfsNewBuilder(void)
}
}
+LIBHDFS_C_API
void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn)
{
errno = 0;
bld->overrideHost = std::string(nn);
}
+LIBHDFS_C_API
void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port)
{
errno = 0;
bld->overridePort = port;
}
+LIBHDFS_C_API
void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName)
{
errno = 0;
@@ -1607,12 +1666,14 @@ void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName)
}
}
+LIBHDFS_C_API
void hdfsBuilderSetForceNewInstance(struct hdfsBuilder *bld) {
//libhdfspp always returns a new instance, so nothing to do
(void)bld;
errno = 0;
}
+LIBHDFS_C_API
void hdfsFreeBuilder(struct hdfsBuilder *bld)
{
try
@@ -1626,6 +1687,7 @@ void hdfsFreeBuilder(struct hdfsBuilder *bld)
}
}
+LIBHDFS_C_API
int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key,
const char *val)
{
@@ -1650,16 +1712,22 @@ int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key,
}
}
+LIBHDFS_C_API
void hdfsConfStrFree(char *val)
{
errno = 0;
free(val);
}
+LIBHDFS_C_API
hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) {
- return doHdfsConnect(bld->overrideHost, bld->overridePort, bld->user, bld->config.GetOptions());
+ hdfsFS fs = doHdfsConnect(bld->overrideHost, bld->overridePort, bld->user, bld->config.GetOptions());
+ // Always free the builder
+ hdfsFreeBuilder(bld);
+ return fs;
}
+LIBHDFS_C_API
int hdfsConfGetStr(const char *key, char **val)
{
try
@@ -1674,6 +1742,7 @@ int hdfsConfGetStr(const char *key, char **val)
}
}
+LIBHDFS_C_API
int hdfsConfGetInt(const char *key, int32_t *val)
{
try
@@ -1706,6 +1775,7 @@ struct hdfsBuilder *hdfsNewBuilderFromDirectory(const char * configDirectory)
}
}
+LIBHDFSPP_EXT_API
int hdfsBuilderConfGetStr(struct hdfsBuilder *bld, const char *key,
char **val)
{
@@ -1739,6 +1809,7 @@ bool isValidInt(int64_t value)
value <= std::numeric_limits<int>::max());
}
+LIBHDFSPP_EXT_API
int hdfsBuilderConfGetInt(struct hdfsBuilder *bld, const char *key, int32_t *val)
{
try
@@ -1765,6 +1836,7 @@ int hdfsBuilderConfGetInt(struct hdfsBuilder *bld, const char *key, int32_t *val
}
}
+LIBHDFSPP_EXT_API
int hdfsBuilderConfGetLong(struct hdfsBuilder *bld, const char *key, int64_t *val)
{
try
@@ -1859,15 +1931,17 @@ void CForwardingLogger::FreeLogData(LogData *data) {
free(data);
}
-
+LIBHDFSPP_EXT_API
LogData *hdfsCopyLogData(LogData *data) {
return CForwardingLogger::CopyLogData(data);
}
+LIBHDFSPP_EXT_API
void hdfsFreeLogData(LogData *data) {
CForwardingLogger::FreeLogData(data);
}
+LIBHDFSPP_EXT_API
void hdfsSetLogFunction(void (*callback)(LogData*)) {
CForwardingLogger *logger = new CForwardingLogger();
logger->SetCallback(callback);
@@ -1900,6 +1974,7 @@ static bool IsComponentValid(int component) {
return true;
}
+LIBHDFSPP_EXT_API
int hdfsEnableLoggingForComponent(int component) {
errno = 0;
if(!IsComponentValid(component))
@@ -1908,6 +1983,7 @@ int hdfsEnableLoggingForComponent(int component) {
return 0;
}
+LIBHDFSPP_EXT_API
int hdfsDisableLoggingForComponent(int component) {
errno = 0;
if(!IsComponentValid(component))
@@ -1916,6 +1992,7 @@ int hdfsDisableLoggingForComponent(int component) {
return 0;
}
+LIBHDFSPP_EXT_API
int hdfsSetLoggingLevel(int level) {
errno = 0;
if(!IsLevelValid(level))
@@ -1923,3 +2000,8 @@ int hdfsSetLoggingLevel(int level) {
LogManager::SetLogLevel(static_cast<LogLevel>(level));
return 0;
}
+
+#undef LIBHDFS_C_API
+#undef LIBHDFSPP_EXT_API
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/48db24a4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.cc
index 9d9a816..e83a28c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.cc
@@ -52,8 +52,10 @@ HANamenodeTracker::HANamenodeTracker(const std::vector<ResolvedNamenodeInfo> &se
active_info_ = servers[0];
standby_info_ = servers[1];
- LOG_INFO(kRPC, << "Active namenode url = " << active_info_.uri.str());
- LOG_INFO(kRPC, << "Standby namenode url = " << standby_info_.uri.str());
+ LOG_INFO(kRPC, << "HA enabled. Using the following namenodes from the configuration."
+ << "\nNote: Active namenode cannot be determined until a connection has been made.")
+ LOG_INFO(kRPC, << "First namenode url = " << active_info_.uri.str());
+ LOG_INFO(kRPC, << "Second namenode url = " << standby_info_.uri.str());
enabled_ = true;
if(!active_info_.endpoints.empty() || !standby_info_.endpoints.empty()) {
@@ -64,51 +66,57 @@ HANamenodeTracker::HANamenodeTracker(const std::vector<ResolvedNamenodeInfo> &se
HANamenodeTracker::~HANamenodeTracker() {}
-// Pass in endpoint from current connection, this will do a reverse lookup
-// and return the info for the standby node. It will also swap its state internally.
-ResolvedNamenodeInfo HANamenodeTracker::GetFailoverAndUpdate(::asio::ip::tcp::endpoint current_endpoint) {
- LOG_TRACE(kRPC, << "Swapping from endpoint " << current_endpoint);
+bool HANamenodeTracker::GetFailoverAndUpdate(const std::vector<::asio::ip::tcp::endpoint>& current_endpoints,
+ ResolvedNamenodeInfo& out)
+{
mutex_guard swap_lock(swap_lock_);
- ResolvedNamenodeInfo failover_node;
+ // Cannot look up without a key.
+ if(current_endpoints.size() == 0) {
+ event_handlers_->call(FS_NN_EMPTY_ENDPOINTS_EVENT, active_info_.nameservice.c_str(),
+ 0 /*Not much to say about context without endpoints*/);
+ LOG_ERROR(kRPC, << "HANamenodeTracker@" << this << "::GetFailoverAndUpdate requires at least 1 endpoint.");
+ return false;
+ }
+
+ LOG_TRACE(kRPC, << "Swapping from endpoint " << current_endpoints[0]);
- // Connected to standby, switch standby to active
- if(IsCurrentActive_locked(current_endpoint)) {
+ if(IsCurrentActive_locked(current_endpoints[0])) {
std::swap(active_info_, standby_info_);
if(event_handlers_)
event_handlers_->call(FS_NN_FAILOVER_EVENT, active_info_.nameservice.c_str(),
reinterpret_cast<int64_t>(active_info_.uri.str().c_str()));
- failover_node = active_info_;
- } else if(IsCurrentStandby_locked(current_endpoint)) {
+ out = active_info_;
+ } else if(IsCurrentStandby_locked(current_endpoints[0])) {
// Connected to standby
if(event_handlers_)
event_handlers_->call(FS_NN_FAILOVER_EVENT, active_info_.nameservice.c_str(),
reinterpret_cast<int64_t>(active_info_.uri.str().c_str()));
- failover_node = active_info_;
+ out = active_info_;
} else {
- // Invalid state, throw for testing
- std::string ep1 = format_endpoints(active_info_.endpoints);
- std::string ep2 = format_endpoints(standby_info_.endpoints);
-
- std::stringstream msg;
- msg << "Looked for " << current_endpoint << " in\n";
- msg << ep1 << " and\n";
- msg << ep2 << std::endl;
-
- LOG_ERROR(kRPC, << "Unable to find RPC connection in config " << msg.str() << ". Bailing out.");
- throw std::runtime_error(msg.str());
+ // Invalid state (or a NIC was added that didn't show up during DNS)
+ std::stringstream errorMsg; // asio specializes endpoing operator<< for stringstream
+ errorMsg << "Unable to find RPC connection in config. Looked for " << current_endpoints[0] << " in\n"
+ << format_endpoints(active_info_.endpoints) << " and\n"
+ << format_endpoints(standby_info_.endpoints) << std::endl;
+ LOG_ERROR(kRPC, << errorMsg.str());
+ return false;
}
- if(failover_node.endpoints.empty()) {
- LOG_WARN(kRPC, << "No endpoints for node " << failover_node.uri.str() << " attempting to resolve again");
- if(!ResolveInPlace(ioservice_, failover_node)) {
- LOG_ERROR(kRPC, << "Fallback endpoint resolution for node " << failover_node.uri.str()
- << "failed. Please make sure your configuration is up to date.");
+ // Extra DNS on swapped node to try and get EPs if it didn't already have them
+ if(out.endpoints.empty()) {
+ LOG_WARN(kRPC, << "No endpoints for node " << out.uri.str() << " attempting to resolve again");
+ if(!ResolveInPlace(ioservice_, out)) {
+ // Stuck retrying against the same NN that was able to be resolved in this case
+ LOG_ERROR(kRPC, << "Fallback endpoint resolution for node " << out.uri.str()
+ << " failed. Please make sure your configuration is up to date.");
}
}
- return failover_node;
+
+ return true;
}
+
bool HANamenodeTracker::IsCurrentActive_locked(const ::asio::ip::tcp::endpoint &ep) const {
for(unsigned int i=0;i<active_info_.endpoints.size();i++) {
if(ep.address() == active_info_.endpoints[i].address()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/48db24a4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.h
index f51e13c..cc34f51 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.h
@@ -48,15 +48,18 @@ class HANamenodeTracker {
bool is_enabled() const { return enabled_; }
bool is_resolved() const { return resolved_; }
- // Get node opposite of the current one if possible (swaps active/standby)
+ // Pass in vector of endpoints held by RpcConnection, use endpoints to infer node
+ // currently being used. Swap internal state and set out to other node.
// Note: This will always mutate internal state. Use IsCurrentActive/Standby to
// get info without changing state
- ResolvedNamenodeInfo GetFailoverAndUpdate(::asio::ip::tcp::endpoint current_endpoint);
+ bool GetFailoverAndUpdate(const std::vector<::asio::ip::tcp::endpoint>& current_endpoints,
+ ResolvedNamenodeInfo& out);
+ private:
+ // See if endpoint ep is part of the list of endpoints for the active or standby NN
bool IsCurrentActive_locked(const ::asio::ip::tcp::endpoint &ep) const;
bool IsCurrentStandby_locked(const ::asio::ip::tcp::endpoint &ep) const;
- private:
// If HA should be enabled, according to our options and runtime info like # nodes provided
bool enabled_;
// If we were able to resolve at least 1 HA namenode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/48db24a4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
index f59f2ce..34af423 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
@@ -213,6 +213,11 @@ void RpcEngine::RpcCommsError(
optional<RetryAction> head_action = optional<RetryAction>();
// Filter out anything with too many retries already
+ if(event_handlers_) {
+ event_handlers_->call(FS_NN_PRE_RPC_RETRY_EVENT, "RpcCommsError",
+ reinterpret_cast<int64_t>(this));
+ }
+
for (auto it = pendingRequests.begin(); it < pendingRequests.end();) {
auto req = *it;
@@ -261,15 +266,34 @@ void RpcEngine::RpcCommsError(
// If HA is enabled and we have valid HA info then fail over to the standby (hopefully now active)
if(head_action->action == RetryAction::FAILOVER_AND_RETRY && ha_persisted_info_) {
- for(unsigned int i=0; i<pendingRequests.size();i++)
+ for(unsigned int i=0; i<pendingRequests.size();i++) {
pendingRequests[i]->IncrementFailoverCount();
+ }
- ResolvedNamenodeInfo new_active_nn_info =
- ha_persisted_info_->GetFailoverAndUpdate(last_endpoints_[0]/*reverse lookup*/);
+ ResolvedNamenodeInfo new_active_nn_info;
+ bool failoverInfoFound = ha_persisted_info_->GetFailoverAndUpdate(last_endpoints_, new_active_nn_info);
+ if(!failoverInfoFound) {
+ // This shouldn't be a common case, the set of endpoints was empty, likely due to DNS issues.
+ // Another possibility is a network device has been added or removed due to a VM starting or stopping.
+
+ LOG_ERROR(kRPC, << "Failed to find endpoints for the alternate namenode."
+ << "Make sure Namenode hostnames can be found with a DNS lookup.");
+ // Kill all pending RPC requests since there's nowhere for this to go
+ Status badEndpointStatus = Status::Error("No endpoints found for namenode");
+
+ for(unsigned int i=0; i<pendingRequests.size(); i++) {
+ std::shared_ptr<Request> sharedCurrentRequest = pendingRequests[i];
+ io_service().post([sharedCurrentRequest, badEndpointStatus]() {
+ sharedCurrentRequest->OnResponseArrived(nullptr, badEndpointStatus); // Never call back while holding a lock
+ });
+ }
- LOG_INFO(kRPC, << "Going to try connecting to alternate Namenode: " << new_active_nn_info.uri.str());
+ // Clear request vector. This isn't a recoverable error.
+ pendingRequests.clear();
+ }
if(ha_persisted_info_->is_resolved()) {
+ LOG_INFO(kRPC, << "Going to try connecting to alternate Namenode: " << new_active_nn_info.uri.str());
last_endpoints_ = new_active_nn_info.endpoints;
} else {
LOG_WARN(kRPC, << "It looks HA is turned on, but unable to fail over. has info="
http://git-wip-us.apache.org/repos/asf/hadoop/blob/48db24a4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
index 0b4581e..3331935 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
@@ -120,6 +120,12 @@ add_executable(user_lock_test user_lock_test.cc)
target_link_libraries(user_lock_test fs gmock_main common ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
add_memcheck_test(user_lock user_lock_test)
+add_executable(hdfs_config_connect_bugs_test hdfs_config_connect_bugs.cc)
+target_link_libraries(hdfs_config_connect_bugs_test common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
+add_memcheck_test(hdfs_config_connect_bugs hdfs_config_connect_bugs_test)
+
+
+
#
#
# INTEGRATION TESTS - TESTS THE FULL LIBRARY AGAINST ACTUAL SERVERS
@@ -136,7 +142,7 @@ include_directories (
add_library(hdfspp_test_shim_static STATIC hdfs_shim.c libhdfs_wrapper.c libhdfspp_wrapper.cc ${LIBHDFSPP_BINDING_C}/hdfs.cc)
-# TODO: get all of the mini dfs library bits here in one plase
+# TODO: get all of the mini dfs library bits here in one place
# add_library(hdfspp_mini_cluster native_mini_dfs ${JAVA_JVM_LIBRARY} )
#TODO: Link against full library rather than just parts
@@ -157,4 +163,8 @@ build_libhdfs_test(hdfs_ext hdfspp_test_shim_static ${CMAKE_CURRENT_LIST_DIR}/hd
link_libhdfs_test (hdfs_ext hdfspp_test_shim_static hdfspp_static gmock_main native_mini_dfs ${JAVA_JVM_LIBRARY} ${SASL_LIBRARIES})
add_libhdfs_test (hdfs_ext hdfspp_test_shim_static)
+#build_libhdfs_test(hdfs_config_connect_bugs hdfspp_test_shim_static ${CMAKE_CURRENT_LIST_DIR}/hdfs_config_connect_bugs.cc)
+#link_libhdfs_test (hdfs_config_connect_bugs hdfspp_test_shim_static hdfspp_static gmock_main native_mini_dfs ${JAVA_JVM_LIBRARY} ${SASL_LIBRARIES})
+#add_libhdfs_test (hdfs_config_connect_bugs hdfspp_test_shim_static)
+
endif(HADOOP_BUILD)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/48db24a4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_config_connect_bugs.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_config_connect_bugs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_config_connect_bugs.cc
new file mode 100644
index 0000000..fc31227
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_config_connect_bugs.cc
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hdfspp/hdfs_ext.h"
+
+#include "configuration_test.h"
+
+#include <google/protobuf/stubs/common.h>
+
+#include <cstring>
+#include <chrono>
+#include <exception>
+
+
+static const char *hdfs_11294_core_site_txt =
+"<configuration>\n"
+" <property name=\"fs.defaultFS\" value=\"hdfs://NAMESERVICE1\"/>\n"
+" <property name=\"hadoop.security.authentication\" value=\"simple\"/>\n"
+" <property name=\"ipc.client.connect.retry.interval\" value=\"1\">\n"
+"</configuration>\n";
+
+static const char *hdfs_11294_hdfs_site_txt =
+"<configuration>\n"
+" <property>\n"
+" <name>dfs.nameservices</name>\n"
+" <value>NAMESERVICE1</value>\n"
+" </property>\n"
+" <property>\n"
+" <name>dfs.ha.namenodes.NAMESERVICE1</name>\n"
+" <value>nn1, nn2</value>\n"
+" </property>\n"
+" <property>\n"
+" <name>dfs.namenode.rpc-address.NAMESERVICE1.nn1</name>\n"
+" <value>nonesuch1.apache.org:8020</value>\n"
+" </property>\n"
+" <property>\n"
+" <name>dfs.namenode.servicerpc-address.NAMESERVICE1.nn1</name>\n"
+" <value>nonesuch1.apache.org:8040</value>\n"
+" </property>\n"
+" <property>\n"
+" <name>dfs.namenode.http-address.NAMESERVICE1.nn1</name>\n"
+" <value>nonesuch1.apache.org:50070</value>\n"
+" </property>\n"
+" <property>\n"
+" <name>dfs.namenode.rpc-address.NAMESERVICE1.nn2</name>\n"
+" <value>nonesuch2.apache.org:8020</value>\n"
+" </property>\n"
+" <property>\n"
+" <name>dfs.namenode.servicerpc-address.NAMESERVICE1.nn2</name>\n"
+" <value>nonesuch2.apache.org:8040</value>\n"
+" </property>\n"
+" <property>\n"
+" <name>dfs.namenode.http-address.NAMESERVICE1.nn2</name>\n"
+" <value>nonesuch2.apache.org:50070</value>\n"
+" </property>\n"
+"</configuration>\n";
+
+
+
+
+namespace hdfs {
+
+// Make sure we can set up a mini-cluster and connect to it
+TEST(ConfigConnectBugs, Test_HDFS_11294) {
+ // Directory for hdfs config
+ TempDir td;
+
+ const std::string& tempDirPath = td.path;
+ const std::string coreSitePath = tempDirPath + "/core-site.xml";
+ const std::string hdfsSitePath = tempDirPath + "/hdfs-site.xml";
+
+ // Write configs
+ FILE *coreSite = fopen(coreSitePath.c_str(), "w");
+ EXPECT_NE(coreSite, nullptr);
+ int coreSiteLength = strlen(hdfs_11294_core_site_txt);
+ size_t res = fwrite(hdfs_11294_core_site_txt, 1, coreSiteLength, coreSite);
+ EXPECT_EQ(res, coreSiteLength);
+ EXPECT_EQ(fclose(coreSite), 0);
+
+ FILE *hdfsSite = fopen(hdfsSitePath.c_str(), "w");
+ EXPECT_NE(hdfsSite, nullptr);
+ int hdfsSiteLength = strlen(hdfs_11294_hdfs_site_txt);
+ res = fwrite(hdfs_11294_hdfs_site_txt, 1, hdfsSiteLength, hdfsSite);
+ EXPECT_EQ(res, hdfsSiteLength);
+ EXPECT_EQ(fclose(hdfsSite), 0);
+
+ // Load configs with new FS
+ hdfsBuilder *bld = hdfsNewBuilderFromDirectory(tempDirPath.c_str());
+ hdfsBuilderSetNameNode(bld, "NAMESERVICE1");
+
+ // In HDFS-11294 connecting would crash because DNS couldn't resolve
+ // endpoints but the RpcEngine would attempt to dereference a non existant
+ // element in a std::vector and crash. Test passes if connect doesn't crash.
+ hdfsFS fileSystem = hdfsBuilderConnect(bld);
+
+ // FS shouldn't be created if it can't connect.
+ EXPECT_EQ(fileSystem, nullptr);
+
+ // Verify it got to endpoint check
+ char errMsgBuf[100];
+ memset(errMsgBuf, 0, 100);
+ EXPECT_EQ( hdfsGetLastError(errMsgBuf, 100), 0);
+ EXPECT_STREQ(errMsgBuf, "Exception:No endpoints found for namenode");
+
+
+ // remove config files
+ EXPECT_EQ(remove(coreSitePath.c_str()), 0);
+ EXPECT_EQ(remove(hdfsSitePath.c_str()), 0);
+}
+
+} // end namespace hdfs
+
+int main(int argc, char *argv[]) {
+ // The following line must be executed to initialize Google Mock
+ // (and Google Test) before running the tests.
+ ::testing::InitGoogleMock(&argc, argv);
+ int exit_code = RUN_ALL_TESTS();
+ google::protobuf::ShutdownProtobufLibrary();
+
+ return exit_code;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/48db24a4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
index 2b6cdbc..776894e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
@@ -398,14 +398,20 @@ TEST(RpcEngineTest, TestEventCallbacks)
io_service.stop();
ASSERT_TRUE(stat.ok());
});
+
+ // If you're adding event hooks you'll most likely need to update this.
+ // It's a brittle test but makes it hard to miss control flow changes in RPC retry.
+ for(const auto& m : callbacks)
+ std::cerr << m << std::endl;
io_service.run();
ASSERT_TRUE(complete);
- ASSERT_EQ(8, callbacks.size());
+ ASSERT_EQ(9, callbacks.size());
ASSERT_EQ(FS_NN_CONNECT_EVENT, callbacks[0]); // error
- ASSERT_EQ(FS_NN_CONNECT_EVENT, callbacks[1]); // reconnect
- ASSERT_EQ(FS_NN_READ_EVENT, callbacks[2]); // makes an error
- ASSERT_EQ(FS_NN_CONNECT_EVENT, callbacks[3]); // reconnect
- for (int i=4; i < 7; i++)
+ ASSERT_EQ(FS_NN_PRE_RPC_RETRY_EVENT, callbacks[1]); // figure out retry decision
+ ASSERT_EQ(FS_NN_CONNECT_EVENT, callbacks[2]); // reconnect
+ ASSERT_EQ(FS_NN_PRE_RPC_RETRY_EVENT, callbacks[3]); // makes an error
+ ASSERT_EQ(FS_NN_CONNECT_EVENT, callbacks[4]); // reconnect
+ for (int i=5; i < 8; i++)
ASSERT_EQ(FS_NN_READ_EVENT, callbacks[i]);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/52] [abbrv] hadoop git commit: HDFS-11971: libhdfs++: [Fix] a few
portability issues. Contributed by Anatoli Schein
Posted by jh...@apache.org.
HDFS-11971: libhdfs++: [Fix] a few portability issues. Contributed by Anatoli Schein
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c5e7a695
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c5e7a695
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c5e7a695
Branch: refs/heads/HDFS-8707
Commit: c5e7a69523f4bce0f09c7afe423452c36213a072
Parents: 1c9b174
Author: James Clampffer <ja...@hp.com>
Authored: Tue Jun 20 15:17:58 2017 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../src/main/native/libhdfspp/CMakeLists.txt | 12 ++++--
.../libhdfspp/examples/c/cat/CMakeLists.txt | 10 +----
.../examples/c/connect_cancel/CMakeLists.txt | 10 +----
.../libhdfspp/examples/cc/cat/CMakeLists.txt | 12 +-----
.../examples/cc/connect_cancel/CMakeLists.txt | 2 +-
.../libhdfspp/examples/cc/find/CMakeLists.txt | 12 +-----
.../examples/cc/gendirs/CMakeLists.txt | 12 +-----
.../native/libhdfspp/lib/common/configuration.h | 2 +-
.../main/native/libhdfspp/tools/CMakeLists.txt | 44 ++++++++++----------
9 files changed, 40 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c5e7a695/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
index a5c77e0..dc7cd30 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
@@ -176,6 +176,7 @@ include_directories( SYSTEM
third_party/uriparser2
${OPENSSL_INCLUDE_DIR}
${SASL_INCLUDE_DIR}
+ ${PROTOBUF_INCLUDE_DIRS}
)
@@ -206,6 +207,7 @@ if (HADOOP_BUILD)
${PROTOBUF_LIBRARY}
${OPENSSL_LIBRARIES}
${SASL_LIBRARIES}
+ ${CMAKE_THREAD_LIBS_INIT}
)
else (HADOOP_BUILD)
add_library(hdfspp_static STATIC ${EMPTY_FILE_CC} ${LIBHDFSPP_ALL_OBJECTS})
@@ -214,6 +216,7 @@ else (HADOOP_BUILD)
${PROTOBUF_LIBRARY}
${OPENSSL_LIBRARIES}
${SASL_LIBRARIES}
+ ${CMAKE_THREAD_LIBS_INIT}
)
add_library(hdfspp SHARED ${EMPTY_FILE_CC} ${LIBHDFSPP_ALL_OBJECTS})
target_link_libraries(hdfspp_static
@@ -221,6 +224,7 @@ else (HADOOP_BUILD)
${PROTOBUF_LIBRARY}
${OPENSSL_LIBRARIES}
${SASL_LIBRARIES}
+ ${CMAKE_THREAD_LIBS_INIT}
)
endif (HADOOP_BUILD)
set(LIBHDFSPP_VERSION "0.1.0")
@@ -231,11 +235,11 @@ set_target_properties(hdfspp PROPERTIES
# Can be installed to a particular location via "make DESTDIR=... install"
file(GLOB_RECURSE LIBHDFSPP_HEADER_FILES "${CMAKE_CURRENT_LIST_DIR}/include/*.h*")
file(GLOB_RECURSE LIBHDFS_HEADER_FILES "${HADOOP_IMPORT_DIR}/include/*.h*")
-install(FILES ${LIBHDFSPP_HEADER_FILES} DESTINATION /include/hdfspp)
-install(FILES ${LIBHDFS_HEADER_FILES} DESTINATION /include/hdfs)
+install(FILES ${LIBHDFSPP_HEADER_FILES} DESTINATION include/hdfspp)
+install(FILES ${LIBHDFS_HEADER_FILES} DESTINATION include/hdfs)
-install(TARGETS hdfspp_static ARCHIVE DESTINATION /lib)
-install(TARGETS hdfspp LIBRARY DESTINATION /lib)
+install(TARGETS hdfspp_static ARCHIVE DESTINATION lib)
+install(TARGETS hdfspp LIBRARY DESTINATION lib)
add_custom_target(
InstallToBuildDirectory
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c5e7a695/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/CMakeLists.txt
index 1319293..41a9ee8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/CMakeLists.txt
@@ -24,12 +24,4 @@ include_directories( ${LIBHDFSPP_DIR}/include )
link_directories( ${LIBHDFSPP_DIR}/lib )
add_executable(cat_c cat.c)
-target_link_libraries(cat_c hdfspp uriparser2)
-
-# Several examples in different languages need to produce executables with
-# same names. To allow executables with same names we keep their CMake
-# names different, but specify their executable names as follows:
-set_target_properties( cat_c
- PROPERTIES
- OUTPUT_NAME "cat"
-)
+target_link_libraries(cat_c hdfspp_static uriparser2)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c5e7a695/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/connect_cancel/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/connect_cancel/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/connect_cancel/CMakeLists.txt
index 56aeeea..6276467 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/connect_cancel/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/connect_cancel/CMakeLists.txt
@@ -24,12 +24,4 @@ include_directories( ${LIBHDFSPP_DIR}/include )
link_directories( ${LIBHDFSPP_DIR}/lib )
add_executable(connect_cancel_c connect_cancel.c)
-target_link_libraries(connect_cancel_c hdfspp uriparser2)
-
-# Several examples in different languages need to produce executables with
-# same names. To allow executables with same names we keep their CMake
-# names different, but specify their executable names as follows:
-set_target_properties( connect_cancel_c
- PROPERTIES
- OUTPUT_NAME "connect_cancel_c"
-)
+target_link_libraries(connect_cancel_c hdfspp_static uriparser2)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c5e7a695/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/CMakeLists.txt
index 9ad919a..479ff86 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/CMakeLists.txt
@@ -23,13 +23,5 @@ set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX})
include_directories( ${LIBHDFSPP_DIR}/include )
link_directories( ${LIBHDFSPP_DIR}/lib )
-add_executable(cat_cc cat.cc)
-target_link_libraries(cat_cc hdfspp)
-
-# Several examples in different languages need to produce executables with
-# same names. To allow executables with same names we keep their CMake
-# names different, but specify their executable names as follows:
-set_target_properties( cat_cc
- PROPERTIES
- OUTPUT_NAME "cat"
-)
+add_executable(cat cat.cc)
+target_link_libraries(cat hdfspp_static)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c5e7a695/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/connect_cancel/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/connect_cancel/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/connect_cancel/CMakeLists.txt
index 6ee44cc..e3cc0b5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/connect_cancel/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/connect_cancel/CMakeLists.txt
@@ -24,4 +24,4 @@ include_directories( ${LIBHDFSPP_DIR}/include )
link_directories( ${LIBHDFSPP_DIR}/lib )
add_executable(connect_cancel connect_cancel.cc)
-target_link_libraries(connect_cancel hdfspp)
+target_link_libraries(connect_cancel hdfspp_static)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c5e7a695/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/find/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/find/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/find/CMakeLists.txt
index e894e58..8210bb5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/find/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/find/CMakeLists.txt
@@ -23,13 +23,5 @@ set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX})
include_directories( ${LIBHDFSPP_DIR}/include )
link_directories( ${LIBHDFSPP_DIR}/lib )
-add_executable(find_cc find.cc)
-target_link_libraries(find_cc hdfspp)
-
-# Several examples in different languages need to produce executables with
-# same names. To allow executables with same names we keep their CMake
-# names different, but specify their executable names as follows:
-set_target_properties( find_cc
- PROPERTIES
- OUTPUT_NAME "find"
-)
+add_executable(find find.cc)
+target_link_libraries(find hdfspp_static)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c5e7a695/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/gendirs/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/gendirs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/gendirs/CMakeLists.txt
index e25f425..7fdc1b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/gendirs/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/gendirs/CMakeLists.txt
@@ -23,13 +23,5 @@ set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX})
include_directories( ${LIBHDFSPP_DIR}/include )
link_directories( ${LIBHDFSPP_DIR}/lib )
-add_executable(gendirs_cc gendirs.cc)
-target_link_libraries(gendirs_cc hdfspp)
-
-# Several examples in different languages need to produce executables with
-# same names. To allow executables with same names we keep their CMake
-# names different, but specify their executable names as follows:
-set_target_properties( gendirs_cc
- PROPERTIES
- OUTPUT_NAME "gendirs"
-)
+add_executable(gendirs gendirs.cc)
+target_link_libraries(gendirs hdfspp_static)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c5e7a695/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.h
index b6b968e..ebfa909 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.h
@@ -98,7 +98,7 @@ protected:
static std::string fixCase(const std::string &in) {
std::string result(in);
- for (auto & c: result) c = toupper(c);
+ for (auto & c: result) c = (char) toupper(c);
return result;
}
};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c5e7a695/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt
index 4a46c7a..dbcb97f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt
@@ -27,67 +27,67 @@ add_library(tools_common_obj OBJECT tools_common.cc)
add_library(tools_common $<TARGET_OBJECTS:tools_common_obj>)
add_executable(hdfs_cat hdfs_cat.cc)
-target_link_libraries(hdfs_cat tools_common hdfspp)
+target_link_libraries(hdfs_cat tools_common hdfspp_static)
add_executable(hdfs_chgrp hdfs_chgrp.cc)
-target_link_libraries(hdfs_chgrp tools_common hdfspp)
+target_link_libraries(hdfs_chgrp tools_common hdfspp_static)
add_executable(hdfs_chown hdfs_chown.cc)
-target_link_libraries(hdfs_chown tools_common hdfspp)
+target_link_libraries(hdfs_chown tools_common hdfspp_static)
add_executable(hdfs_chmod hdfs_chmod.cc)
-target_link_libraries(hdfs_chmod tools_common hdfspp)
+target_link_libraries(hdfs_chmod tools_common hdfspp_static)
add_executable(hdfs_find hdfs_find.cc)
-target_link_libraries(hdfs_find tools_common hdfspp)
+target_link_libraries(hdfs_find tools_common hdfspp_static)
add_executable(hdfs_mkdir hdfs_mkdir.cc)
-target_link_libraries(hdfs_mkdir tools_common hdfspp)
+target_link_libraries(hdfs_mkdir tools_common hdfspp_static)
add_executable(hdfs_rm hdfs_rm.cc)
-target_link_libraries(hdfs_rm tools_common hdfspp)
+target_link_libraries(hdfs_rm tools_common hdfspp_static)
add_executable(hdfs_ls hdfs_ls.cc)
-target_link_libraries(hdfs_ls tools_common hdfspp)
+target_link_libraries(hdfs_ls tools_common hdfspp_static)
add_executable(hdfs_stat hdfs_stat.cc)
-target_link_libraries(hdfs_stat tools_common hdfspp)
+target_link_libraries(hdfs_stat tools_common hdfspp_static)
add_executable(hdfs_count hdfs_count.cc)
-target_link_libraries(hdfs_count tools_common hdfspp)
+target_link_libraries(hdfs_count tools_common hdfspp_static)
add_executable(hdfs_df hdfs_df.cc)
-target_link_libraries(hdfs_df tools_common hdfspp)
+target_link_libraries(hdfs_df tools_common hdfspp_static)
add_executable(hdfs_du hdfs_du.cc)
-target_link_libraries(hdfs_du tools_common hdfspp)
+target_link_libraries(hdfs_du tools_common hdfspp_static)
add_executable(hdfs_get hdfs_get.cc)
-target_link_libraries(hdfs_get tools_common hdfspp)
+target_link_libraries(hdfs_get tools_common hdfspp_static)
add_executable(hdfs_copyToLocal hdfs_copyToLocal.cc)
-target_link_libraries(hdfs_copyToLocal tools_common hdfspp)
+target_link_libraries(hdfs_copyToLocal tools_common hdfspp_static)
add_executable(hdfs_moveToLocal hdfs_moveToLocal.cc)
-target_link_libraries(hdfs_moveToLocal tools_common hdfspp)
+target_link_libraries(hdfs_moveToLocal tools_common hdfspp_static)
add_executable(hdfs_setrep hdfs_setrep.cc)
-target_link_libraries(hdfs_setrep tools_common hdfspp)
+target_link_libraries(hdfs_setrep tools_common hdfspp_static)
add_executable(hdfs_allowSnapshot hdfs_allowSnapshot.cc)
-target_link_libraries(hdfs_allowSnapshot tools_common hdfspp)
+target_link_libraries(hdfs_allowSnapshot tools_common hdfspp_static)
add_executable(hdfs_disallowSnapshot hdfs_disallowSnapshot.cc)
-target_link_libraries(hdfs_disallowSnapshot tools_common hdfspp)
+target_link_libraries(hdfs_disallowSnapshot tools_common hdfspp_static)
add_executable(hdfs_createSnapshot hdfs_createSnapshot.cc)
-target_link_libraries(hdfs_createSnapshot tools_common hdfspp)
+target_link_libraries(hdfs_createSnapshot tools_common hdfspp_static)
add_executable(hdfs_renameSnapshot hdfs_renameSnapshot.cc)
-target_link_libraries(hdfs_renameSnapshot tools_common hdfspp)
+target_link_libraries(hdfs_renameSnapshot tools_common hdfspp_static)
add_executable(hdfs_deleteSnapshot hdfs_deleteSnapshot.cc)
-target_link_libraries(hdfs_deleteSnapshot tools_common hdfspp)
+target_link_libraries(hdfs_deleteSnapshot tools_common hdfspp_static)
add_executable(hdfs_tail hdfs_tail.cc)
-target_link_libraries(hdfs_tail tools_common hdfspp)
\ No newline at end of file
+target_link_libraries(hdfs_tail tools_common hdfspp_static)
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/52] [abbrv] hadoop git commit: HDFS-10931: libhdfs++: Fix object
lifecycle issues in the BlockReader (see follow up work in jira). Contributed
by James Clampffer.
Posted by jh...@apache.org.
HDFS-10931: libhdfs++: Fix object lifecycle issues in the BlockReader (see follow up work in jira). Contributed by James Clampffer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2a42eeb6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2a42eeb6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2a42eeb6
Branch: refs/heads/HDFS-8707
Commit: 2a42eeb66f7fbf3fb4fc434480a712c53cf0243a
Parents: 69cb05d
Author: James <jh...@apache.org>
Authored: Wed Oct 5 11:55:06 2016 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../libhdfs-tests/test_libhdfs_mini_stress.c | 7 +-
.../native/libhdfspp/lib/reader/block_reader.cc | 208 +++++++++++--------
2 files changed, 125 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a42eeb6/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c
index 0d01e44..dca4782 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c
@@ -231,7 +231,12 @@ static int testHdfsMiniStressImpl(struct tlhThreadInfo *ti)
fprintf(stderr, "testHdfsMiniStress(threadIdx=%d): starting\n",
ti->threadIdx);
EXPECT_NONNULL(ti->hdfs);
- EXPECT_ZERO(doTestHdfsMiniStress(ti, 1));
+ // Error injection on, some failures are expected in the read path.
+ // The expectation is that any memory stomps will cascade and cause
+ // the following test to fail. Ideally RPC errors would be seperated
+ // from BlockReader errors (RPC is expected to recover from disconnects).
+ doTestHdfsMiniStress(ti, 1);
+ // No error injection
EXPECT_ZERO(doTestHdfsMiniStress(ti, 0));
return 0;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a42eeb6/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc
index 6098b9c..b63ef76 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc
@@ -30,11 +30,12 @@ namespace hdfs {
#define FMT_CONT_AND_READER_ADDR "this=" << (void*)this << ", reader=" << (void*)reader_
#define FMT_THIS_ADDR "this=" << (void*)this
-hadoop::hdfs::OpReadBlockProto
-ReadBlockProto(const std::string &client_name, bool verify_checksum,
- const hadoop::common::TokenProto *token,
- const hadoop::hdfs::ExtendedBlockProto *block, uint64_t length,
- uint64_t offset) {
+
+// Stuff an OpReadBlockProto message with required fields.
+hadoop::hdfs::OpReadBlockProto ReadBlockProto(const std::string &client_name,
+ bool verify_checksum, const hadoop::common::TokenProto *token,
+ const hadoop::hdfs::ExtendedBlockProto *block, uint64_t length, uint64_t offset)
+{
using namespace hadoop::hdfs;
using namespace hadoop::common;
BaseHeaderProto *base_h = new BaseHeaderProto();
@@ -55,13 +56,28 @@ ReadBlockProto(const std::string &client_name, bool verify_checksum,
return p;
}
+//
+// Notes about the BlockReader and associated object lifecycles (9/29/16)
+// -We have a several stages in the read pipeline. Each stage represents a logical
+// step in the HDFS block transfer logic. They are implemented as continuations
+// for now, and in some cases the stage may have a nested continuation as well.
+// It's important to make sure that continuations, nested or otherwise, cannot
+// outlive the objects they depend on.
+//
+// -The BlockReader holds a shared_ptr to the DataNodeConnection that's used in each
+// pipeline stage. The connection object must never be destroyed while operations are
+// pending on the ASIO side (see HDFS-10931). In order to prevent a state where the
+// BlockReader or one of the corresponding pipelines outlives the connection each
+// pipeline stage must explicitly hold a shared pointer copied from BlockReaderImpl::dn_.
+//
+
static int8_t unsecured_request_block_header[3] = {0, kDataTransferVersion, Operation::kReadBlock};
-void BlockReaderImpl::AsyncRequestBlock(
- const std::string &client_name,
+void BlockReaderImpl::AsyncRequestBlock(const std::string &client_name,
const hadoop::hdfs::ExtendedBlockProto *block, uint64_t length,
- uint64_t offset, const std::function<void(Status)> &handler) {
+ uint64_t offset, const std::function<void(Status)> &handler)
+{
LOG_TRACE(kBlockReader, << "BlockReaderImpl::AsyncRequestBlock("
<< FMT_THIS_ADDR << ", ..., length="
<< length << ", offset=" << offset << ", ...) called");
@@ -95,24 +111,22 @@ void BlockReaderImpl::AsyncRequestBlock(
}
auto read_pb_message =
- new continuation::ReadDelimitedPBMessageContinuation<AsyncStream, 16384>(
- dn_, &s->response);
+ new continuation::ReadDelimitedPBMessageContinuation<AsyncStream, 16384>(dn_, &s->response);
- m->Push(asio_continuation::Write(dn_, asio::buffer(s->header)))
- .Push(read_pb_message);
+ m->Push(asio_continuation::Write(dn_, asio::buffer(s->header))).Push(read_pb_message);
m->Run([this, handler, offset](const Status &status, const State &s) { Status stat = status;
if (stat.ok()) {
const auto &resp = s.response;
- if(this->event_handlers_) {
- event_response event_resp = this->event_handlers_->call(FILE_DN_READ_EVENT, "", "", 0);
+ if(this->event_handlers_) {
+ event_response event_resp = this->event_handlers_->call(FILE_DN_READ_EVENT, "", "", 0);
#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
- if (stat.ok() && event_resp.response() == event_response::kTest_Error) {
- stat = Status::Error("Test error");
- }
+ if (stat.ok() && event_resp.response() == event_response::kTest_Error) {
+ stat = Status::Error("Test error");
+ }
#endif
- }
+ }
if (stat.ok() && resp.status() == ::hadoop::hdfs::Status::SUCCESS) {
if (resp.has_readopchecksuminfo()) {
@@ -128,10 +142,10 @@ void BlockReaderImpl::AsyncRequestBlock(
});
}
-Status BlockReaderImpl::RequestBlock(
- const std::string &client_name,
- const hadoop::hdfs::ExtendedBlockProto *block, uint64_t length,
- uint64_t offset) {
+Status BlockReaderImpl::RequestBlock(const std::string &client_name,
+ const hadoop::hdfs::ExtendedBlockProto *block,
+ uint64_t length, uint64_t offset)
+{
LOG_TRACE(kBlockReader, << "BlockReaderImpl::RequestBlock("
<< FMT_THIS_ADDR <<"..., length="
<< length << ", offset=" << offset << ") called");
@@ -143,15 +157,9 @@ Status BlockReaderImpl::RequestBlock(
return future.get();
}
-hadoop::hdfs::OpReadBlockProto
-ReadBlockProto(const std::string &client_name, bool verify_checksum,
- const hadoop::common::TokenProto *token,
- const hadoop::hdfs::ExtendedBlockProto *block, uint64_t length,
- uint64_t offset);
-
-struct BlockReaderImpl::ReadPacketHeader
- : continuation::Continuation {
- ReadPacketHeader(BlockReaderImpl *parent) : parent_(parent) {}
+struct BlockReaderImpl::ReadPacketHeader : continuation::Continuation
+{
+ ReadPacketHeader(BlockReaderImpl *parent) : parent_(parent), shared_conn_(parent->dn_) {}
virtual void Run(const Next &next) override {
LOG_TRACE(kBlockReader, << "BlockReaderImpl::ReadPacketHeader::Run("
@@ -184,8 +192,7 @@ struct BlockReaderImpl::ReadPacketHeader
asio::async_read(*parent_->dn_, asio::buffer(buf_),
std::bind(&ReadPacketHeader::CompletionHandler, this,
- std::placeholders::_1, std::placeholders::_2),
- handler);
+ std::placeholders::_1, std::placeholders::_2), handler);
}
private:
@@ -216,10 +223,14 @@ private:
return kHeaderStart + header_length() - transferred;
}
}
+
+ // Keep the DN connection alive
+ std::shared_ptr<DataNodeConnection> shared_conn_;
};
-struct BlockReaderImpl::ReadChecksum : continuation::Continuation {
- ReadChecksum(BlockReaderImpl *parent) : parent_(parent) {}
+struct BlockReaderImpl::ReadChecksum : continuation::Continuation
+{
+ ReadChecksum(BlockReaderImpl *parent) : parent_(parent), shared_conn_(parent->dn_) {}
virtual void Run(const Next &next) override {
LOG_TRACE(kBlockReader, << "BlockReaderImpl::ReadChecksum::Run("
@@ -231,13 +242,15 @@ struct BlockReaderImpl::ReadChecksum : continuation::Continuation {
return;
}
- auto handler = [parent, next, this](const asio::error_code &ec, size_t) {
+ std::shared_ptr<DataNodeConnection> keep_conn_alive_ = shared_conn_;
+
+ auto handler = [parent, next, this, keep_conn_alive_](const asio::error_code &ec, size_t)
+ {
Status status;
if (ec) {
status = Status(ec.value(), ec.message().c_str());
} else {
- parent->state_ =
- parent->chunk_padding_bytes_ ? kReadPadding : kReadData;
+ parent->state_ = parent->chunk_padding_bytes_ ? kReadPadding : kReadData;
}
if(parent->event_handlers_) {
event_response event_resp = parent->event_handlers_->call(FILE_DN_READ_EVENT, "", "", 0);
@@ -249,20 +262,25 @@ struct BlockReaderImpl::ReadChecksum : continuation::Continuation {
}
next(status);
};
- parent->checksum_.resize(parent->packet_len_ - sizeof(int) -
- parent->header_.datalen());
+
+ parent->checksum_.resize(parent->packet_len_ - sizeof(int) - parent->header_.datalen());
+
asio::async_read(*parent->dn_, asio::buffer(parent->checksum_), handler);
}
private:
BlockReaderImpl *parent_;
+
+ // Keep the DataNodeConnection alive
+ std::shared_ptr<DataNodeConnection> shared_conn_;
};
-struct BlockReaderImpl::ReadData : continuation::Continuation {
- ReadData(BlockReaderImpl *parent,
- std::shared_ptr<size_t> bytes_transferred,
- const asio::mutable_buffers_1 &buf)
- : parent_(parent), bytes_transferred_(bytes_transferred), buf_(buf) {
+struct BlockReaderImpl::ReadData : continuation::Continuation
+{
+ ReadData(BlockReaderImpl *parent, std::shared_ptr<size_t> bytes_transferred,
+ const asio::mutable_buffers_1 &buf) : parent_(parent),
+ bytes_transferred_(bytes_transferred), buf_(buf), shared_conn_(parent->dn_)
+ {
buf_.begin();
}
@@ -279,12 +297,15 @@ struct BlockReaderImpl::ReadData : continuation::Continuation {
if (ec) {
status = Status(ec.value(), ec.message().c_str());
}
+
*bytes_transferred_ += transferred;
parent_->bytes_to_read_ -= transferred;
parent_->packet_data_read_bytes_ += transferred;
+
if (parent_->packet_data_read_bytes_ >= parent_->header_.datalen()) {
parent_->state_ = kReadPacketHeader;
}
+
if(parent_->event_handlers_) {
event_response event_resp = parent_->event_handlers_->call(FILE_DN_READ_EVENT, "", "", 0);
#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
@@ -296,24 +317,27 @@ struct BlockReaderImpl::ReadData : continuation::Continuation {
next(status);
};
- auto data_len =
- parent_->header_.datalen() - parent_->packet_data_read_bytes_;
- asio::async_read(*parent_->dn_, buf_, asio::transfer_exactly(data_len),
- handler);
+ auto data_len = parent_->header_.datalen() - parent_->packet_data_read_bytes_;
+
+ asio::async_read(*parent_->dn_, buf_, asio::transfer_exactly(data_len), handler);
}
private:
BlockReaderImpl *parent_;
std::shared_ptr<size_t> bytes_transferred_;
const asio::mutable_buffers_1 buf_;
+
+ // Keep DNConnection alive.
+ std::shared_ptr<DataNodeConnection> shared_conn_;
};
-struct BlockReaderImpl::ReadPadding : continuation::Continuation {
- ReadPadding(BlockReaderImpl *parent)
- : parent_(parent), padding_(parent->chunk_padding_bytes_),
+struct BlockReaderImpl::ReadPadding : continuation::Continuation
+{
+ ReadPadding(BlockReaderImpl *parent) : parent_(parent),
+ padding_(parent->chunk_padding_bytes_),
bytes_transferred_(std::make_shared<size_t>(0)),
- read_data_(new ReadData(
- parent, bytes_transferred_, asio::buffer(padding_))) {}
+ read_data_(new ReadData(parent, bytes_transferred_, asio::buffer(padding_))),
+ shared_conn_(parent->dn_) {}
virtual void Run(const Next &next) override {
LOG_TRACE(kBlockReader, << "BlockReaderImpl::ReadPadding::Run("
@@ -324,11 +348,12 @@ struct BlockReaderImpl::ReadPadding : continuation::Continuation {
return;
}
- auto h = [next, this](const Status &stat) {
+ std::shared_ptr<DataNodeConnection> keep_conn_alive_ = shared_conn_;
+
+ auto h = [next, this, keep_conn_alive_](const Status &stat) {
Status status = stat;
if (status.ok()) {
- assert(reinterpret_cast<const int &>(*bytes_transferred_) ==
- parent_->chunk_padding_bytes_);
+ assert(reinterpret_cast<const int &>(*bytes_transferred_) == parent_->chunk_padding_bytes_);
parent_->chunk_padding_bytes_ = 0;
parent_->state_ = kReadData;
}
@@ -352,11 +377,15 @@ private:
std::shared_ptr<continuation::Continuation> read_data_;
ReadPadding(const ReadPadding &) = delete;
ReadPadding &operator=(const ReadPadding &) = delete;
+
+ // Keep DNConnection alive.
+ std::shared_ptr<DataNodeConnection> shared_conn_;
};
-struct BlockReaderImpl::AckRead : continuation::Continuation {
- AckRead(BlockReaderImpl *parent) : parent_(parent) {}
+struct BlockReaderImpl::AckRead : continuation::Continuation
+{
+ AckRead(BlockReaderImpl *parent) : parent_(parent), shared_conn_(parent->dn_) {}
virtual void Run(const Next &next) override {
LOG_TRACE(kBlockReader, << "BlockReaderImpl::AckRead::Run(" << FMT_CONT_AND_PARENT_ADDR << ") called");
@@ -366,17 +395,18 @@ struct BlockReaderImpl::AckRead : continuation::Continuation {
return;
}
- auto m =
- continuation::Pipeline<hadoop::hdfs::ClientReadStatusProto>::Create(parent_->cancel_state_);
+ auto m = continuation::Pipeline<hadoop::hdfs::ClientReadStatusProto>::Create(parent_->cancel_state_);
+
m->state().set_status(parent_->options_.verify_checksum
? hadoop::hdfs::Status::CHECKSUM_OK
: hadoop::hdfs::Status::SUCCESS);
- m->Push(
- continuation::WriteDelimitedPBMessage(parent_->dn_, &m->state()));
+ m->Push(continuation::WriteDelimitedPBMessage(parent_->dn_, &m->state()));
+
+ std::shared_ptr<DataNodeConnection> keep_conn_alive_ = shared_conn_;
- m->Run([this, next](const Status &stat,
- const hadoop::hdfs::ClientReadStatusProto &) {
+ m->Run([this, next, keep_conn_alive_](const Status &stat, const hadoop::hdfs::ClientReadStatusProto &)
+ {
Status status = stat;
if (status.ok()) {
parent_->state_ = BlockReaderImpl::kFinished;
@@ -395,11 +425,14 @@ struct BlockReaderImpl::AckRead : continuation::Continuation {
private:
BlockReaderImpl *parent_;
+
+ // Keep DNConnection alive.
+ std::shared_ptr<DataNodeConnection> shared_conn_;
};
-void BlockReaderImpl::AsyncReadPacket(
- const MutableBuffers &buffers,
- const std::function<void(const Status &, size_t bytes_transferred)> &handler) {
+void BlockReaderImpl::AsyncReadPacket(const MutableBuffers &buffers,
+ const std::function<void(const Status &, size_t bytes_transferred)> &handler)
+{
assert(state_ != kOpen && "Not connected");
LOG_TRACE(kBlockReader, << "BlockReaderImpl::AsyncReadPacket called");
@@ -407,9 +440,11 @@ void BlockReaderImpl::AsyncReadPacket(
struct State {
std::shared_ptr<size_t> bytes_transferred;
};
+
auto m = continuation::Pipeline<State>::Create(cancel_state_);
m->state().bytes_transferred = std::make_shared<size_t>(0);
+ // Note: some of these continuations have nested pipelines.
m->Push(new ReadPacketHeader(this))
.Push(new ReadChecksum(this))
.Push(new ReadPadding(this))
@@ -424,9 +459,8 @@ void BlockReaderImpl::AsyncReadPacket(
}
-size_t
-BlockReaderImpl::ReadPacket(const MutableBuffers &buffers,
- Status *status) {
+size_t BlockReaderImpl::ReadPacket(const MutableBuffers &buffers, Status *status)
+{
LOG_TRACE(kBlockReader, << "BlockReaderImpl::ReadPacket called");
size_t transferred = 0;
@@ -443,12 +477,12 @@ BlockReaderImpl::ReadPacket(const MutableBuffers &buffers,
}
-struct BlockReaderImpl::RequestBlockContinuation : continuation::Continuation {
+struct BlockReaderImpl::RequestBlockContinuation : continuation::Continuation
+{
RequestBlockContinuation(BlockReader *reader, const std::string &client_name,
- const hadoop::hdfs::ExtendedBlockProto *block,
- uint64_t length, uint64_t offset)
- : reader_(reader), client_name_(client_name), length_(length),
- offset_(offset) {
+ const hadoop::hdfs::ExtendedBlockProto *block, uint64_t length, uint64_t offset)
+ : reader_(reader), client_name_(client_name), length_(length), offset_(offset)
+ {
block_.CheckTypeAndMergeFrom(*block);
}
@@ -456,8 +490,7 @@ struct BlockReaderImpl::RequestBlockContinuation : continuation::Continuation {
LOG_TRACE(kBlockReader, << "BlockReaderImpl::RequestBlockContinuation::Run("
<< FMT_CONT_AND_READER_ADDR << ") called");
- reader_->AsyncRequestBlock(client_name_, &block_, length_,
- offset_, next);
+ reader_->AsyncRequestBlock(client_name_, &block_, length_, offset_, next);
}
private:
@@ -468,12 +501,10 @@ private:
uint64_t offset_;
};
-struct BlockReaderImpl::ReadBlockContinuation : continuation::Continuation {
- ReadBlockContinuation(BlockReader *reader, MutableBuffers buffer,
- size_t *transferred)
- : reader_(reader), buffer_(buffer),
- buffer_size_(asio::buffer_size(buffer)), transferred_(transferred) {
- }
+struct BlockReaderImpl::ReadBlockContinuation : continuation::Continuation
+{
+ ReadBlockContinuation(BlockReader *reader, MutableBuffers buffer, size_t *transferred)
+ : reader_(reader), buffer_(buffer), buffer_size_(asio::buffer_size(buffer)), transferred_(transferred) {}
virtual void Run(const Next &next) override {
LOG_TRACE(kBlockReader, << "BlockReaderImpl::ReadBlockContinuation::Run("
@@ -511,7 +542,8 @@ void BlockReaderImpl::AsyncReadBlock(
const hadoop::hdfs::LocatedBlockProto &block,
size_t offset,
const MutableBuffers &buffers,
- const std::function<void(const Status &, size_t)> handler) {
+ const std::function<void(const Status &, size_t)> handler)
+{
LOG_TRACE(kBlockReader, << "BlockReaderImpl::AsyncReadBlock("
<< FMT_THIS_ADDR << ") called");
@@ -520,12 +552,10 @@ void BlockReaderImpl::AsyncReadBlock(
size_t size = asio::buffer_size(buffers);
- m->Push(new RequestBlockContinuation(this, client_name,
- &block.b(), size, offset))
+ m->Push(new RequestBlockContinuation(this, client_name, &block.b(), size, offset))
.Push(new ReadBlockContinuation(this, buffers, bytesTransferred));
- m->Run([handler] (const Status &status,
- const size_t totalBytesTransferred) {
+ m->Run([handler] (const Status &status, const size_t totalBytesTransferred) {
handler(status, totalBytesTransferred);
});
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/52] [abbrv] hadoop git commit: HDFS-10450: libhdfs++: Add Cyrus
SASL support. Contributed by Don Davis.
Posted by jh...@apache.org.
HDFS-10450: libhdfs++: Add Cyrus SASL support. Contributed by Don Davis.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/549a5dbc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/549a5dbc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/549a5dbc
Branch: refs/heads/HDFS-8707
Commit: 549a5dbce9f6aa398908e29e05c0a7e85f3cdc23
Parents: 9d1c902
Author: Bob Hansen <bo...@hpe.com>
Authored: Fri Sep 9 14:31:53 2016 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
dev-support/docker/Dockerfile | 1 -
.../src/main/native/libhdfspp/CMakeLists.txt | 67 +++--
.../native/libhdfspp/lib/rpc/CMakeLists.txt | 13 +-
.../native/libhdfspp/lib/rpc/rpc_connection.cc | 2 +-
.../native/libhdfspp/lib/rpc/sasl_engine.cc | 159 ++---------
.../main/native/libhdfspp/lib/rpc/sasl_engine.h | 79 ++----
.../native/libhdfspp/lib/rpc/sasl_protocol.cc | 266 ++++++++++++-------
.../native/libhdfspp/lib/rpc/sasl_protocol.h | 22 +-
8 files changed, 297 insertions(+), 312 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/549a5dbc/dev-support/docker/Dockerfile
----------------------------------------------------------------------
diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile
index 9eb9bcf..c93c78e 100644
--- a/dev-support/docker/Dockerfile
+++ b/dev-support/docker/Dockerfile
@@ -201,4 +201,3 @@ ENV FORREST_HOME=/opt/apache-forrest
ADD hadoop_env_checks.sh /root/hadoop_env_checks.sh
RUN chmod 755 /root/hadoop_env_checks.sh
RUN echo '~/hadoop_env_checks.sh' >> /root/.bashrc
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/549a5dbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
index da56403..e0476cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
@@ -23,13 +23,15 @@ cmake_minimum_required(VERSION 2.8)
enable_testing()
include (CTest)
SET(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH})
-SET(CMAKE_DISABLE_FIND_PACKAGE_CyrusSASL TRUE) # Until development is done.
+
+# If there's a better way to inform FindCyrusSASL.cmake, let's make this cleaner:
+SET(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH};${CYRUS_SASL_DIR};${GSASL_DIR}")
find_package(Doxygen)
find_package(OpenSSL REQUIRED)
find_package(Protobuf REQUIRED)
-find_package(GSasl)
find_package(CyrusSASL)
+find_package(GSasl)
find_package(Threads)
find_program(MEMORYCHECK_COMMAND valgrind HINTS ${VALGRIND_DIR} )
@@ -41,36 +43,47 @@ if (REQUIRE_VALGRIND AND MEMORYCHECK_COMMAND MATCHES "MEMORYCHECK_COMMAND-NOTFOU
"The path can be included via a -DVALGRIND_DIR=... flag passed to CMake.")
endif (REQUIRE_VALGRIND AND MEMORYCHECK_COMMAND MATCHES "MEMORYCHECK_COMMAND-NOTFOUND" )
+# Find the SASL library to use. If you don't want to require a sasl library,
+# define -DNO_SASL=1 in your cmake call
# Prefer Cyrus SASL, but use GSASL if it is found
# Note that the packages can be disabled by setting CMAKE_DISABLE_FIND_PACKAGE_GSasl or
# CMAKE_DISABLE_FIND_PACKAGE_CyrusSASL, respectively (case sensitive)
set (SASL_LIBRARIES)
set (SASL_INCLUDE_DIR)
-if (CYRUS_SASL_FOUND)
- message(STATUS "Using Cyrus SASL; link with ${CYRUS_SASL_LIBRARIES}")
- set (SASL_INCLUDE_DIR ${CYRUS_SASL_INCLUDE_DIR})
- set (SASL_LIBRARIES ${CYRUS_SASL_SHARED_LIB})
- add_definitions(-DUSE_SASL -DUSE_CYRUS_SASL)
-else (CYRUS_SASL_FOUND)
- if (REQUIRE_CYRUS_SASL)
- message(FATAL_ERROR "Cyrus SASL was required but not found. "
- "The path can be included via a -DCYRUS_SASL_DIR=... flag passed to CMake.")
- endif (REQUIRE_CYRUS_SASL)
-
- # If we didn't pick Cyrus, use GSASL instead
- if (GSASL_FOUND)
- message(STATUS "Using GSASL; link with ${GSASL_LIBRARIES}")
- set (SASL_INCLUDE_DIR ${GSASL_INCLUDE_DIR})
- set (SASL_LIBRARIES ${GSASL_LIBRARIES})
- add_definitions(-DUSE_SASL -DUSE_GSASL)
- else (GSASL_FOUND)
- if (REQUIRE_GSASL)
- message(FATAL_ERROR "GSASL was required but not found. "
- "The path can be included via a -DGSASL_DIR=... flag passed to CMake.")
- endif (REQUIRE_GSASL)
- message(STATUS "Not using SASL")
- endif (GSASL_FOUND)
-endif (CYRUS_SASL_FOUND)
+if (NOT NO_SASL)
+ if (CYRUS_SASL_FOUND)
+ message(STATUS "Using Cyrus SASL; link with ${CYRUS_SASL_SHARED_LIB}")
+ set (SASL_INCLUDE_DIR ${CYRUS_SASL_INCLUDE_DIR})
+ set (SASL_LIBRARIES ${CYRUS_SASL_SHARED_LIB})
+ set (CMAKE_USING_CYRUS_SASL 1)
+ add_definitions(-DUSE_SASL -DUSE_CYRUS_SASL)
+ else (CYRUS_SASL_FOUND)
+ if (REQUIRE_CYRUS_SASL)
+ message(FATAL_ERROR "Cyrus SASL was required but not found. "
+ "The path can be included via a -DCYRUS_SASL_DIR=... flag passed to CMake.")
+ endif (REQUIRE_CYRUS_SASL)
+
+ # If we didn't pick Cyrus, use GSASL instead
+ if (GSASL_FOUND)
+ message(STATUS "Using GSASL; link with ${GSASL_LIBRARIES}")
+ set (SASL_INCLUDE_DIR ${GSASL_INCLUDE_DIR})
+ set (SASL_LIBRARIES ${GSASL_LIBRARIES})
+ set (CMAKE_USING_GSASL 1)
+ add_definitions(-DUSE_SASL -DUSE_GSASL)
+ else (GSASL_FOUND)
+ if (REQUIRE_GSASL)
+ message(FATAL_ERROR "GSASL was required but not found. "
+ "The path can be included via a -DGSASL_DIR=... flag passed to CMake.")
+ endif (REQUIRE_GSASL)
+
+ # No SASL was found, but NO_SASL was not defined
+ message(FATAL_ERROR "Cound not find a SASL library (GSASL (gsasl) or Cyrus SASL (libsasl2). "
+ "Install/configure one of them or define NO_SASL=1 in your cmake call")
+ endif (GSASL_FOUND)
+ endif (CYRUS_SASL_FOUND)
+else (NOT NO_SASL)
+ message(STATUS "Compiling with NO SASL SUPPORT")
+endif (NOT NO_SASL)
add_definitions(-DASIO_STANDALONE -DASIO_CPP11_DATE_TIME)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/549a5dbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt
index 8c80633..84debdd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt
@@ -16,7 +16,16 @@
# limitations under the License.
#
-include_directories(${OPENSSL_INCLUDE_DIRS})
-add_library(rpc_obj OBJECT rpc_connection.cc rpc_engine.cc sasl_protocol.cc sasl_engine.cc)
+list(APPEND rpc_object_items rpc_connection.cc rpc_engine.cc sasl_protocol.cc sasl_engine.cc)
+if (CMAKE_USING_CYRUS_SASL)
+ list(APPEND rpc_object_items cyrus_sasl_engine.cc)
+endif (CMAKE_USING_CYRUS_SASL)
+if (CMAKE_USING_GSASL)
+ list(APPEND rpc_object_items gsasl_engine.cc)
+endif (CMAKE_USING_GSASL)
+
+add_library(rpc_obj OBJECT ${rpc_object_items})
+
+
add_dependencies(rpc_obj proto)
add_library(rpc $<TARGET_OBJECTS:rpc_obj>)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/549a5dbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
index fccfb09..b6136c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
@@ -214,7 +214,7 @@ void RpcConnection::HandshakeComplete(const Status &s) {
#ifdef USE_SASL
sasl_protocol_ = std::make_shared<SaslProtocol>(cluster_name_, auth_info_, shared_from_this());
sasl_protocol_->SetEventHandlers(event_handlers_);
- sasl_protocol_->authenticate([shared_this, this](
+ sasl_protocol_->Authenticate([shared_this, this](
const Status & status, const AuthInfo & new_auth_info) {
AuthComplete(status, new_auth_info); } );
#else
http://git-wip-us.apache.org/repos/asf/hadoop/blob/549a5dbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_engine.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_engine.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_engine.cc
index 3bb067a..3cebe7e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_engine.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_engine.cc
@@ -16,20 +16,19 @@
* limitations under the License.
*/
-#include "sasl_engine.h"
-
-#include "common/logging.h"
-
#include <sstream>
+#include <string.h> // memcpy()
+#include "sasl_engine.h"
+#include "common/logging.h"
namespace hdfs {
/*****************************************************************************
- * BASE CLASS
+ * SASL ENGINE BASE CLASS
*/
-SaslEngine::State SaslEngine::getState()
+SaslEngine::State SaslEngine::GetState()
{
return state_;
}
@@ -37,149 +36,45 @@ SaslEngine::State SaslEngine::getState()
SaslEngine::~SaslEngine() {
}
-Status SaslEngine::setKerberosInfo(const std::string &principal)
+Status SaslEngine::SetKerberosInfo(const std::string &principal)
{
principal_ = principal;
return Status::OK();
}
-Status SaslEngine::setPasswordInfo(const std::string &id,
- const std::string &password)
+Status SaslEngine::SetPasswordInfo(const std::string &id,
+ const std::string &password)
{
id_ = id;
password_ = password;
return Status::OK();
}
+bool SaslEngine::ChooseMech(const std::vector<SaslMethod> &resp_auths) {
+ Status status = Status::OK();
-#ifdef USE_GSASL
-/*****************************************************************************
- * GSASL
- */
-
-#include <gsasl.h>
+ if (resp_auths.empty()) return NULL;
+ for (SaslMethod auth: resp_auths) {
+ if ( auth.mechanism != "GSSAPI") continue; // Hack: only GSSAPI for now
-/*****************************************************************************
- * UTILITY FUNCTIONS
- */
+ // do a proper deep copy of the vector element
+ // that we like, because the original v ector will go away:
+ chosen_mech_.mechanism = auth.mechanism;
+ chosen_mech_.protocol = auth.protocol;
+ chosen_mech_.serverid = auth.serverid;
+ chosen_mech_.challenge = auth.challenge;
-Status gsasl_rc_to_status(int rc)
-{
- if (rc == GSASL_OK) {
- return Status::OK();
- } else {
- std::ostringstream ss;
- ss << "Cannot initialize client (" << rc << "): " << gsasl_strerror(rc);
- return Status::Error(ss.str().c_str());
+ return auth.mechanism.c_str();
}
-}
+ state_ = kErrorState;
+ status = Status::Error("SaslEngine::chooseMech(): No good protocol.");
-GSaslEngine::~GSaslEngine()
-{
- if (session_ != nullptr) {
- gsasl_finish(session_);
- }
+ // Clear out the chosen mech
+ chosen_mech_ = SaslMethod();
- if (ctx_ != nullptr) {
- gsasl_done(ctx_);
- }
-}
+ return NULL;
+} // choose_mech()
-std::pair<Status, SaslMethod> GSaslEngine::start(const std::vector<SaslMethod> &protocols)
-{
- int rc = gsasl_init(&ctx_);
- if (rc != GSASL_OK) {
- state_ = kError;
- return std::make_pair(gsasl_rc_to_status(rc), SaslMethod());
- }
-
- // Hack to only do GSSAPI at the moment
- for (auto protocol: protocols) {
- if (protocol.mechanism == "GSSAPI") {
- Status init = init_kerberos(protocol);
- if (init.ok()) {
- state_ = kWaitingForData;
- return std::make_pair(init, protocol);
- } else {
- state_ = kError;
- return std::make_pair(init, SaslMethod());
- }
- }
- }
-
- state_ = kError;
- return std::make_pair(Status::Error("No good protocol"), SaslMethod());
-}
-
-Status GSaslEngine::init_kerberos(const SaslMethod & mechanism) {
- /* Create new authentication session. */
- int rc = gsasl_client_start(ctx_, mechanism.mechanism.c_str(), &session_);
- if (rc != GSASL_OK) {
- return gsasl_rc_to_status(rc);
- }
-
- if (!principal_) {
- return Status::Error("Attempted kerberos authentication with no principal");
- }
-
- gsasl_property_set(session_, GSASL_SERVICE, mechanism.protocol.c_str());
- gsasl_property_set(session_, GSASL_AUTHID, principal_.value().c_str());
- gsasl_property_set(session_, GSASL_HOSTNAME, mechanism.serverid.c_str());
- return Status::OK();
- }
-
- std::pair<Status, std::string> GSaslEngine::step(const std::string data)
- {
- if (state_ != kWaitingForData)
- LOG_WARN(kRPC, << "GSaslEngine::step when state is " << state_);
-
- char * output = NULL;
- size_t outputSize;
- int rc = gsasl_step(session_, data.c_str(), data.size(), &output,
- &outputSize);
-
- if (rc == GSASL_NEEDS_MORE || rc == GSASL_OK) {
- std::string retval(output, output ? outputSize : 0);
- if (output) {
- free(output);
- }
-
- if (rc == GSASL_OK) {
- state_ = kSuccess;
- }
-
- return std::make_pair(Status::OK(), retval);
- }
- else {
- if (output) {
- free(output);
- }
- state_ = kFailure;
- return std::make_pair(gsasl_rc_to_status(rc), "");
- }
- }
-
-Status GSaslEngine::finish()
-{
- if (state_ != kSuccess && state_ != kFailure && state_ != kError )
- LOG_WARN(kRPC, << "GSaslEngine::finish when state is " << state_);
-
- if (session_ != nullptr) {
- gsasl_finish(session_);
- session_ = NULL;
- }
-
- if (ctx_ != nullptr) {
- gsasl_done(ctx_);
- ctx_ = nullptr;
- }
-
- return Status::OK();
-}
-#endif // USE_GSASL
-
-
-
-}
+} // namespace hdfs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/549a5dbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_engine.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_engine.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_engine.h
index e38c30e..53ec75b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_engine.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_engine.h
@@ -22,14 +22,12 @@
#include "hdfspp/status.h"
#include "optional.hpp"
-#ifdef USE_GSASL
-#include "gsasl.h"
-#endif
-
#include <vector>
namespace hdfs {
+class SaslProtocol;
+
template <class T>
using optional = std::experimental::optional<T>;
@@ -38,7 +36,7 @@ public:
std::string protocol;
std::string mechanism;
std::string serverid;
- void * data;
+ std::string challenge;
};
class SaslEngine {
@@ -48,7 +46,7 @@ public:
kWaitingForData,
kSuccess,
kFailure,
- kError,
+ kErrorState,
};
// State transitions:
@@ -56,70 +54,51 @@ public:
// kUnstarted --start--> kWaitingForData --step-+--> kSuccess --finish--v
// \-> kFailure -/
- SaslEngine() : state_ (kUnstarted) {}
+ // State transitions:
+ // \--------------------------/
+ // kUnstarted --start--> kWaitingForData --step-+--> kSuccess --finish--v
+ // \-> kFailure -/
+
+ SaslEngine(): state_ (kUnstarted) {}
virtual ~SaslEngine();
// Must be called when state is kUnstarted
- Status setKerberosInfo(const std::string &principal);
+ Status SetKerberosInfo(const std::string &principal);
// Must be called when state is kUnstarted
- Status setPasswordInfo(const std::string &id,
+ Status SetPasswordInfo(const std::string &id,
const std::string &password);
+ // Choose a mechanism from the available ones. Will set the
+ // chosen_mech_ member and return true if we found one we
+ // can process
+ bool ChooseMech(const std::vector<SaslMethod> &avail_auths);
+
// Returns the current state
- State getState();
+ State GetState();
// Must be called when state is kUnstarted
- virtual std::pair<Status,SaslMethod> start(
- const std::vector<SaslMethod> &protocols) = 0;
+ virtual std::pair<Status,std::string> Start() = 0;
// Must be called when state is kWaitingForData
// Returns kOK and any data that should be sent to the server
- virtual std::pair<Status,std::string> step(const std::string data) = 0;
+ virtual std::pair<Status,std::string> Step(const std::string data) = 0;
- // Must only be called when state is kSuccess, kFailure, or kError
- virtual Status finish() = 0;
+ // Must only be called when state is kSuccess, kFailure, or kErrorState
+ virtual Status Finish() = 0;
+
+ // main repository of generic Sasl config data:
+ SaslMethod chosen_mech_;
protected:
State state_;
+ SaslProtocol * sasl_protocol_;
optional<std::string> principal_;
+ optional<std::string> realm_;
optional<std::string> id_;
optional<std::string> password_;
-};
+}; // class SaslEngine
-#ifdef USE_GSASL
-class GSaslEngine : public SaslEngine
-{
-public:
- GSaslEngine() : SaslEngine(), ctx_(nullptr), session_(nullptr) {}
- virtual ~GSaslEngine();
-
- virtual std::pair<Status,SaslMethod> start(
- const std::vector<SaslMethod> &protocols);
- virtual std::pair<Status,std::string> step(const std::string data);
- virtual Status finish();
-private:
- Gsasl * ctx_;
- Gsasl_session * session_;
-
- Status init_kerberos(const SaslMethod & mechanism);
-};
-#endif
-
-#ifdef USE_CYRUS_SASL
-class CyrusSaslEngine : public SaslEngine
-{
-public:
- GSaslEngine() : SaslEngine(), ctx_(nullptr), session_(nullptr) {}
- virtual ~GSaslEngine();
-
- virtual std::pair<Status,SaslMethod> start(
- const std::vector<SaslMethod> &protocols);
- virtual std::pair<Status,std::string> step(const std::string data);
- virtual Status finish();
-private:
-};
-#endif
+} // namespace hdfs
-}
#endif /* LIB_RPC_SASLENGINE_H */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/549a5dbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
index 31be77c..b35a2f9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
@@ -16,11 +16,22 @@
* limitations under the License.
*/
-#include "sasl_protocol.h"
-#include "sasl_engine.h"
#include "rpc_engine.h"
#include "common/logging.h"
+#include "sasl_engine.h"
+#include "sasl_protocol.h"
+
+#if defined USE_SASL
+ #if defined USE_CYRUS_SASL
+ #include "cyrus_sasl_engine.h" // CySaslEngine()
+ #elif defined USE_GSASL
+ #include "gsasl_engine.h" // GSaslEngine()
+ #else
+ #error USE_SASL defined but no engine (USE_GSASL) defined
+ #endif
+#endif
+
#include <optional.hpp>
namespace hdfs {
@@ -60,17 +71,13 @@ SaslProtocol::~SaslProtocol()
void SaslProtocol::SetEventHandlers(std::shared_ptr<LibhdfsEvents> event_handlers) {
std::lock_guard<std::mutex> state_lock(sasl_state_lock_);
event_handlers_ = event_handlers;
-}
+ event_handlers_->call("SASL Start", cluster_name_.c_str(), 0);
+} // SetEventHandlers() method
-void SaslProtocol::authenticate(std::function<void(const Status & status, const AuthInfo new_auth_info)> callback)
+void SaslProtocol::Authenticate(std::function<void(const Status & status, const AuthInfo new_auth_info)> callback)
{
std::lock_guard<std::mutex> state_lock(sasl_state_lock_);
- LOG_TRACE(kRPC, << "Authenticating as " << auth_info_.getUser());
-
- assert(state_ == kUnstarted);
- event_handlers_->call("SASL Start", cluster_name_.c_str(), 0);
-
callback_ = callback;
state_ = kNegotiate;
@@ -86,8 +93,9 @@ void SaslProtocol::authenticate(std::function<void(const Status & status, const
std::shared_ptr<RpcSaslProto> resp_msg = std::make_shared<RpcSaslProto>();
auto self(shared_from_this());
connection->AsyncRpc_locked(SASL_METHOD_NAME, req_msg.get(), resp_msg,
- [self, req_msg, resp_msg] (const Status & status) { self->OnServerResponse(status, resp_msg.get()); } );
-}
+ [self, req_msg, resp_msg] (const Status & status) {
+ self->OnServerResponse(status, resp_msg.get()); } );
+} // authenticate() method
AuthInfo::AuthMethod ParseMethod(const std::string & method)
{
@@ -103,113 +111,195 @@ AuthInfo::AuthMethod ParseMethod(const std::string & method)
else {
return AuthInfo::kUnknownAuth;
}
-}
+} // ParseMethod()
-void SaslProtocol::Negotiate(const hadoop::common::RpcSaslProto * response)
+// build_init_msg():
+// Helper function for Start(), to keep
+// these ProtoBuf-RPC calls out of the sasl_engine code.
+
+std::pair<Status, RpcSaslProto>
+SaslProtocol::BuildInitMessage(std::string & token, const hadoop::common::RpcSaslProto * negotiate_msg)
{
- std::vector<SaslMethod> protocols;
+ // The init message needs one of the RpcSaslProto_SaslAuth structures that
+ // was sent in the negotiate message as our chosen mechanism
+ // Map the chosen_mech name back to the RpcSaslProto_SaslAuth from the negotiate
+ // message that it corresponds to
+ SaslMethod & chosenMech = sasl_engine_->chosen_mech_;
+ auto auths = negotiate_msg->auths();
+ auto pb_auth_it = std::find_if(auths.begin(), auths.end(),
+ [chosenMech](const RpcSaslProto_SaslAuth & data)
+ {
+ return data.mechanism() == chosenMech.mechanism;
+ });
+ if (pb_auth_it == auths.end())
+ return std::make_pair(Status::Error("Couldn't find mechanism in negotiate msg"), RpcSaslProto());
+ auto & pb_auth = *pb_auth_it;
+
+ // Prepare INITIATE message
+ RpcSaslProto initiate = RpcSaslProto();
+
+ initiate.set_state(RpcSaslProto_SaslState_INITIATE);
+ // initiate message will contain:
+ // token_ (binary data), and
+ // auths_[ ], an array of objects just like pb_auth.
+ // In our case, we want the auths array
+ // to hold just the single element from pb_auth:
+
+ RpcSaslProto_SaslAuth * respAuth = initiate.add_auths();
+ respAuth->CopyFrom(pb_auth);
+
+ // Mostly, an INITIATE message contains a "Token".
+ // For GSSAPI, the token is a Kerberos AP_REQ, aka
+ // "Authenticated application request," comprising
+ // the client's application ticket & and an encrypted
+ // message that Kerberos calls an "authenticator".
+
+ if (token.empty()) {
+ const char * errmsg = "SaslProtocol::build_init_msg(): No token available.";
+ LOG_ERROR(kRPC, << errmsg);
+ return std::make_pair(Status::Error(errmsg), RpcSaslProto());
+ }
- bool simple_available = false;
+ // add challenge token to the INITIATE message:
+ initiate.set_token(token);
-#if defined USE_SASL
- #if defined USE_CYRUS_SASL
- sasl_engine_.reset(new CyrusSaslEngine());
- #elif defined USE_GSASL
- sasl_engine_.reset(new GSaslEngine());
- #else
- #error USE_SASL defined but no engine (USE_GSASL) defined
- #endif
-#endif
- if (auth_info_.getToken()) {
- sasl_engine_->setPasswordInfo(auth_info_.getToken().value().identifier,
- auth_info_.getToken().value().password);
- }
- sasl_engine_->setKerberosInfo(auth_info_.getUser()); // HDFS-10451 will look up principal by username
+ // the initiate message is ready to send:
+ return std::make_pair(Status::OK(), initiate);
+} // build_init_msg()
+
+// Converts the RpcSaslProto.auths ararray from RpcSaslProto_SaslAuth PB
+// structures to SaslMethod structures
+static bool
+extract_auths(std::vector<SaslMethod> & resp_auths,
+ const hadoop::common::RpcSaslProto * response) {
+ bool simple_avail = false;
+ auto pb_auths = response->auths();
- auto auths = response->auths();
- for (int i = 0; i < auths.size(); ++i) {
- auto auth = auths.Get(i);
- AuthInfo::AuthMethod method = ParseMethod(auth.method());
+ // For our GSSAPI case, an element of pb_auths contains:
+ // method_ = "KERBEROS"
+ // mechanism_ = "GSSAPI"
+ // protocol_ = "nn" /* "name node", AKA "hdfs"
+ // serverid_ = "foobar1.acmecorp.com"
+ // challenge_ = ""
+ // _cached_size_ = 0
+ // _has_bits_ = 15
+
+ for (int i = 0; i < pb_auths.size(); ++i) {
+ auto pb_auth = pb_auths.Get(i);
+ AuthInfo::AuthMethod method = ParseMethod(pb_auth.method());
switch(method) {
case AuthInfo::kToken:
case AuthInfo::kKerberos: {
SaslMethod new_method;
- new_method.mechanism = auth.mechanism();
- new_method.protocol = auth.protocol();
- new_method.serverid = auth.serverid();
- new_method.data = const_cast<RpcSaslProto_SaslAuth *>(&response->auths().Get(i));
- protocols.push_back(new_method);
+ new_method.mechanism = pb_auth.mechanism();
+ new_method.protocol = pb_auth.protocol();
+ new_method.serverid = pb_auth.serverid();
+ new_method.challenge = pb_auth.has_challenge() ?
+ pb_auth.challenge() : "";
+ resp_auths.push_back(new_method);
}
break;
case AuthInfo::kSimple:
- simple_available = true;
+ simple_avail = true;
break;
case AuthInfo::kUnknownAuth:
- LOG_WARN(kRPC, << "Unknown auth method " << auth.method() << "; ignoring");
+ LOG_WARN(kRPC, << "Unknown auth method " << pb_auth.method() << "; ignoring");
break;
default:
LOG_WARN(kRPC, << "Invalid auth type: " << method << "; ignoring");
break;
}
- }
-
- if (!protocols.empty()) {
- auto init = sasl_engine_->start(protocols);
- if (init.first.ok()) {
- auto chosen_auth = reinterpret_cast<RpcSaslProto_SaslAuth *>(init.second.data);
-
- // Prepare initiate message
- RpcSaslProto initiate;
- initiate.set_state(RpcSaslProto_SaslState_INITIATE);
- RpcSaslProto_SaslAuth * respAuth = initiate.add_auths();
- respAuth->CopyFrom(*chosen_auth);
-
- LOG_TRACE(kRPC, << "Using auth: " << chosen_auth->protocol() << "/" <<
- chosen_auth->mechanism() << "/" << chosen_auth->serverid());
+ } // for
+ return simple_avail;
+} // extract_auths()
- std::string challenge = chosen_auth->has_challenge() ? chosen_auth->challenge() : "";
- auto sasl_challenge = sasl_engine_->step(challenge);
+void SaslProtocol::ResetEngine() {
+#if defined USE_SASL
+ #if defined USE_CYRUS_SASL
+ sasl_engine_.reset(new CySaslEngine());
+ #elif defined USE_GSASL
+ sasl_engine_.reset(new GSaslEngine());
+ #else
+ #error USE_SASL defined but no engine (USE_GSASL) defined
+ #endif
+#endif
+ return;
+} // Reset_Engine() method
- if (sasl_challenge.first.ok()) {
- if (!sasl_challenge.second.empty()) {
- initiate.set_token(sasl_challenge.second);
- }
+void SaslProtocol::Negotiate(const hadoop::common::RpcSaslProto * response)
+{
+ this->ResetEngine(); // get a new SaslEngine
- std::shared_ptr<RpcSaslProto> return_msg = std::make_shared<RpcSaslProto>();
- SendSaslMessage(initiate);
- return;
- } else {
- AuthComplete(sasl_challenge.first, auth_info_);
- return;
- }
- } else if (!simple_available) {
- // If simple IS available, fall through to below
- AuthComplete(init.first, auth_info_);
+ if (auth_info_.getToken()) {
+ sasl_engine_->SetPasswordInfo(auth_info_.getToken().value().identifier,
+ auth_info_.getToken().value().password);
+ }
+ sasl_engine_->SetKerberosInfo(auth_info_.getUser()); // TODO: map to principal?
+
+ // Copy the response's auths list to an array of SaslMethod objects.
+ // SaslEngine shouldn't need to know about the protobuf classes.
+ std::vector<SaslMethod> resp_auths;
+ bool simple_available = extract_auths(resp_auths, response);
+ bool mech_chosen = sasl_engine_->ChooseMech(resp_auths);
+
+ if (mech_chosen) {
+
+ // Prepare an INITIATE message,
+ // later on we'll send it to the hdfs server:
+ auto start_result = sasl_engine_->Start();
+ Status status = start_result.first;
+ if (! status.ok()) {
+ // start() failed, simple isn't avail,
+ // so give up & stop authentication:
+ AuthComplete(status, auth_info_);
return;
}
+ // token.second is a binary buffer, containing
+ // client credentials that will prove the
+ // client's identity to the application server.
+ // Put the token into an INITIATE message:
+ auto init = BuildInitMessage(start_result.second, response);
+
+ // If all is OK, send the INITIATE msg to the hdfs server;
+ // Otherwise, if possible, fail over to simple authentication:
+ status = init.first;
+ if (status.ok()) {
+ SendSaslMessage(init.second);
+ return;
+ }
+ if (!simple_available) {
+ // build_init_msg() failed, simple isn't avail,
+ // so give up & stop authentication:
+ AuthComplete(status, auth_info_);
+ return;
+ }
+ // If simple IS available, fall through to below,
+ // but without build_init_msg()'s failure-status.
}
- // There were no protocols, or the SaslEngine couldn't make one work
+ // There were no resp_auths, or the SaslEngine couldn't make one work
if (simple_available) {
// Simple was the only one we could use. That's OK.
AuthComplete(Status::OK(), auth_info_);
return;
} else {
- // We didn't understand any of the protocols; give back some information
+ // We didn't understand any of the resp_auths;
+ // Give back some information
std::stringstream ss;
ss << "Client cannot authenticate via: ";
- for (int i = 0; i < auths.size(); ++i) {
- auto auth = auths.Get(i);
- ss << auth.mechanism() << ", ";
+ auto pb_auths = response->auths();
+ for (int i = 0; i < pb_auths.size(); ++i) {
+ const RpcSaslProto_SaslAuth & pb_auth = pb_auths.Get(i);
+ ss << pb_auth.mechanism() << ", ";
}
AuthComplete(Status::Error(ss.str().c_str()), auth_info_);
return;
}
-}
+} // Negotiate() method
void SaslProtocol::Challenge(const hadoop::common::RpcSaslProto * challenge)
{
@@ -223,7 +313,7 @@ void SaslProtocol::Challenge(const hadoop::common::RpcSaslProto * challenge)
response.set_state(RpcSaslProto_SaslState_RESPONSE);
std::string challenge_token = challenge->has_token() ? challenge->token() : "";
- auto sasl_response = sasl_engine_->step(challenge_token);
+ auto sasl_response = sasl_engine_->Step(challenge_token);
if (sasl_response.first.ok()) {
response.set_token(sasl_response.second);
@@ -234,7 +324,7 @@ void SaslProtocol::Challenge(const hadoop::common::RpcSaslProto * challenge)
AuthComplete(sasl_response.first, auth_info_);
return;
}
-}
+} // Challenge() method
bool SaslProtocol::SendSaslMessage(RpcSaslProto & message)
{
@@ -256,8 +346,9 @@ bool SaslProtocol::SendSaslMessage(RpcSaslProto & message)
} );
return true;
-}
+} // SendSaslMessage() method
+// AuthComplete(): stop the auth effort, successful ot not:
bool SaslProtocol::AuthComplete(const Status & status, const AuthInfo & auth_info)
{
assert(lock_held(sasl_state_lock_)); // Must be holding lock before calling
@@ -270,15 +361,11 @@ bool SaslProtocol::AuthComplete(const Status & status, const AuthInfo & auth_inf
return false;
}
- if (!status.ok()) {
- auth_info_.setMethod(AuthInfo::kAuthFailed);
- }
-
- LOG_TRACE(kRPC, << "AuthComplete: " << status.ToString());
+ LOG_TRACE(kRPC, << "Received SASL response" << status.ToString());
connection->AuthComplete(status, auth_info);
return true;
-}
+} // AuthComplete() method
void SaslProtocol::OnServerResponse(const Status & status, const hadoop::common::RpcSaslProto * response)
{
@@ -295,7 +382,7 @@ void SaslProtocol::OnServerResponse(const Status & status, const hadoop::common:
break;
case RpcSaslProto_SaslState_SUCCESS:
if (sasl_engine_) {
- sasl_engine_->finish();
+ sasl_engine_->Finish();
}
AuthComplete(Status::OK(), auth_info_);
break;
@@ -314,7 +401,6 @@ void SaslProtocol::OnServerResponse(const Status & status, const hadoop::common:
} else {
AuthComplete(status, auth_info_);
}
-}
+} // OnServerResponse() method
-
-}
+} // namespace hdfs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/549a5dbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.h
index 68191b3..3816cf9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.h
@@ -19,22 +19,23 @@
#ifndef LIB_RPC_SASLPROTOCOL_H
#define LIB_RPC_SASLPROTOCOL_H
-#include "hdfspp/status.h"
-#include "common/auth_info.h"
-#include "common/libhdfs_events_impl.h"
-
-#include <RpcHeader.pb.h>
-
#include <memory>
#include <mutex>
#include <functional>
+#include <RpcHeader.pb.h>
+
+#include "hdfspp/status.h"
+#include "common/auth_info.h"
+#include "common/libhdfs_events_impl.h"
+
namespace hdfs {
static constexpr const char * SASL_METHOD_NAME = "sasl message";
class RpcConnection;
class SaslEngine;
+class SaslMethod;
class SaslProtocol : public std::enable_shared_from_this<SaslProtocol>
{
@@ -48,8 +49,9 @@ public:
// Start the async authentication process. Must be called while holding the
// connection lock, but all callbacks will occur outside of the connection lock
- void authenticate(std::function<void(const Status & status, const AuthInfo new_auth_info)> callback);
+ void Authenticate(std::function<void(const Status & status, const AuthInfo new_auth_info)> callback);
void OnServerResponse(const Status & status, const hadoop::common::RpcSaslProto * response);
+ std::pair<Status, hadoop::common::RpcSaslProto> BuildInitMessage( std::string & token, const hadoop::common::RpcSaslProto * negotiate_msg);
private:
enum State {
kUnstarted,
@@ -72,10 +74,12 @@ private:
bool SendSaslMessage(hadoop::common::RpcSaslProto & message);
bool AuthComplete(const Status & status, const AuthInfo & auth_info);
+ void ResetEngine();
void Negotiate(const hadoop::common::RpcSaslProto * response);
void Challenge(const hadoop::common::RpcSaslProto * response);
-};
-}
+}; // class SaslProtocol
+
+} // namespace hdfs
#endif /* LIB_RPC_SASLPROTOCOL_H */
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/52] [abbrv] hadoop git commit: HDFS-11028: libhdfs++: FileSystem
needs to be able to cancel pending connections. Contributed by James
Clampffer
Posted by jh...@apache.org.
HDFS-11028: libhdfs++: FileSystem needs to be able to cancel pending connections. Contributed by James Clampffer
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8783461e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8783461e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8783461e
Branch: refs/heads/HDFS-8707
Commit: 8783461e2ec3aff2630ea3574a69beb1d5c61e84
Parents: 58de2df
Author: James <jh...@apache.org>
Authored: Mon Jan 23 18:19:13 2017 -0500
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../native/libhdfspp/examples/c/CMakeLists.txt | 1 +
.../examples/c/connect_cancel/CMakeLists.txt | 35 ++++
.../examples/c/connect_cancel/connect_cancel.c | 107 +++++++++++++
.../libhdfspp/examples/cpp/CMakeLists.txt | 1 +
.../examples/cpp/connect_cancel/CMakeLists.txt | 29 ++++
.../cpp/connect_cancel/connect_cancel.cc | 158 +++++++++++++++++++
.../native/libhdfspp/include/hdfspp/hdfs_ext.h | 31 ++++
.../native/libhdfspp/include/hdfspp/hdfspp.h | 6 +
.../native/libhdfspp/lib/bindings/c/hdfs.cc | 108 +++++++++++++
.../libhdfspp/lib/common/continuation/asio.h | 86 ----------
.../lib/common/continuation/continuation.h | 12 --
.../libhdfspp/lib/common/hdfs_configuration.cc | 25 ++-
.../libhdfspp/lib/common/namenode_info.cc | 123 ++++++++++-----
.../src/main/native/libhdfspp/lib/common/util.h | 68 ++++++++
.../main/native/libhdfspp/lib/fs/filesystem.cc | 42 ++++-
.../main/native/libhdfspp/lib/fs/filesystem.h | 8 +
.../libhdfspp/lib/fs/namenode_operations.cc | 4 +
.../libhdfspp/lib/fs/namenode_operations.h | 2 +
.../main/native/libhdfspp/lib/rpc/rpc_engine.cc | 21 ++-
.../main/native/libhdfspp/lib/rpc/rpc_engine.h | 5 +
20 files changed, 729 insertions(+), 143 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8783461e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/CMakeLists.txt
index 76880cd..a73d2bc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/CMakeLists.txt
@@ -17,3 +17,4 @@
#
add_subdirectory(cat)
+add_subdirectory(connect_cancel)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8783461e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/connect_cancel/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/connect_cancel/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/connect_cancel/CMakeLists.txt
new file mode 100644
index 0000000..56aeeea
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/connect_cancel/CMakeLists.txt
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Default LIBHDFSPP_DIR to the default install location. You can override
+# it by add -DLIBHDFSPP_DIR=... to your cmake invocation
+set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX})
+
+include_directories( ${LIBHDFSPP_DIR}/include )
+link_directories( ${LIBHDFSPP_DIR}/lib )
+
+add_executable(connect_cancel_c connect_cancel.c)
+target_link_libraries(connect_cancel_c hdfspp uriparser2)
+
+# Several examples in different languages need to produce executables with
+# same names. To allow executables with same names we keep their CMake
+# names different, but specify their executable names as follows:
+set_target_properties( connect_cancel_c
+ PROPERTIES
+ OUTPUT_NAME "connect_cancel_c"
+)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8783461e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/connect_cancel/connect_cancel.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/connect_cancel/connect_cancel.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/connect_cancel/connect_cancel.c
new file mode 100644
index 0000000..f6af6d1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/connect_cancel/connect_cancel.c
@@ -0,0 +1,107 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+/*
+ Attempt to connect to a cluster and use Control-C to bail out if it takes a while.
+ Valid config must be in environment variable $HADOOP_CONF_DIR
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+#include <unistd.h>
+
+#include "hdfspp/hdfs_ext.h"
+#include "common/util_c.h"
+
+#define ERROR_BUFFER_SIZE 1024
+
+// Global so signal handler can get at it
+hdfsFS fs = NULL;
+
+const char *catch_enter = "In signal handler, going to try and cancel.\n";
+const char *catch_cancel = "hdfsCancelPendingConnect has been canceled in the signal handler.\n";
+const char *catch_exit = "Exiting the signal handler.\n";
+
+// Print to stdout without calling malloc or otherwise indirectly modify userspace state.
+// Write calls to stdout may still interleave with stuff coming from elsewhere.
+static void sighandler_direct_stdout(const char *msg) {
+ if(!msg)
+ return;
+ ssize_t res = write(1 /*posix stdout fd*/, msg, strlen(msg));
+ (void)res;
+}
+
+static void sig_catch(int val) {
+ // Beware of calling things that aren't reentrant e.g. malloc while in a signal handler.
+ sighandler_direct_stdout(catch_enter);
+
+ if(fs) {
+ hdfsCancelPendingConnection(fs);
+ sighandler_direct_stdout(catch_cancel);
+ }
+ sighandler_direct_stdout(catch_exit);
+}
+
+
+int main(int argc, char** argv) {
+ hdfsSetLoggingLevel(HDFSPP_LOG_LEVEL_INFO);
+ signal(SIGINT, sig_catch);
+
+ char error_text[ERROR_BUFFER_SIZE];
+ if (argc != 1) {
+ fprintf(stderr, "usage: ./connect_cancel_c\n");
+ ShutdownProtobufLibrary_C();
+ exit(EXIT_FAILURE);
+ }
+
+ const char *hdfsconfdir = getenv("HADOOP_CONF_DIR");
+ if(!hdfsconfdir) {
+ fprintf(stderr, "$HADOOP_CONF_DIR must be set\n");
+ ShutdownProtobufLibrary_C();
+ exit(EXIT_FAILURE);
+ }
+
+ struct hdfsBuilder* builder = hdfsNewBuilderFromDirectory(hdfsconfdir);
+
+ fs = hdfsAllocateFileSystem(builder);
+ if (fs == NULL) {
+ hdfsGetLastError(error_text, ERROR_BUFFER_SIZE);
+ fprintf(stderr, "hdfsAllocateFileSystem returned null.\n%s\n", error_text);
+ hdfsFreeBuilder(builder);
+ ShutdownProtobufLibrary_C();
+ exit(EXIT_FAILURE);
+ }
+
+ int connected = hdfsConnectAllocated(fs, builder);
+ if (connected != 0) {
+ hdfsGetLastError(error_text, ERROR_BUFFER_SIZE);
+ fprintf(stderr, "hdfsConnectAllocated errored.\n%s\n", error_text);
+ hdfsFreeBuilder(builder);
+ ShutdownProtobufLibrary_C();
+ exit(EXIT_FAILURE);
+ }
+
+ hdfsDisconnect(fs);
+ hdfsFreeBuilder(builder);
+ // Clean up static data and prevent valgrind memory leaks
+ ShutdownProtobufLibrary_C();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8783461e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/CMakeLists.txt
index 9e16b0b..8893f03 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/CMakeLists.txt
@@ -19,3 +19,4 @@
add_subdirectory(cat)
add_subdirectory(gendirs)
add_subdirectory(find)
+add_subdirectory(connect_cancel)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8783461e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/connect_cancel/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/connect_cancel/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/connect_cancel/CMakeLists.txt
new file mode 100644
index 0000000..a029a99
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/connect_cancel/CMakeLists.txt
@@ -0,0 +1,29 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Default LIBHDFSPP_DIR to the default install location. You can override
+# it by add -DLIBHDFSPP_DIR=... to your cmake invocation
+set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX})
+
+include_directories( ${LIBHDFSPP_DIR}/include )
+link_directories( ${LIBHDFSPP_DIR}/lib )
+
+add_executable(connect_cancel connect_cancel.cc)
+target_link_libraries(connect_cancel hdfspp)
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8783461e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/connect_cancel/connect_cancel.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/connect_cancel/connect_cancel.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/connect_cancel/connect_cancel.cc
new file mode 100644
index 0000000..7e5daa9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/connect_cancel/connect_cancel.cc
@@ -0,0 +1,158 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+
+#include "hdfspp/hdfspp.h"
+#include "common/hdfs_configuration.h"
+#include "common/configuration_loader.h"
+
+#include <google/protobuf/stubs/common.h>
+
+#include <signal.h>
+#include <unistd.h>
+
+#include <thread>
+#include <iostream>
+
+// Simple example of how to cancel an async connect call.
+// Here Control-C (SIGINT) is caught in order to invoke the FS level cancel and
+// properly tear down the process. Valgrind should show no leaked memory on exit
+// when cancel has been called. URI parsing code is omitted and defaultFs from
+// /etc/hadoop/conf or $HADOOP_CONF_DIR is always used.
+
+// Scoped globally to make it simple to reference from the signal handler.
+std::shared_ptr<hdfs::FileSystem> fs;
+
+const std::string catch_enter("In signal handler, going to try and cancel FileSystem::Connect.\n");
+const std::string catch_cancel("FileSystem::Cancel has been canceled in the signal handler.\n");
+const std::string catch_exit("Exiting the signal handler.\n");
+
+// Avoid IO reentrancy issues, see comments in signal handler below.
+// It's possible that the write interleaves with another write call,
+// but it won't corrupt the stack or heap.
+static void sighandler_direct_stdout(const std::string &msg) {
+ ssize_t res = ::write(1 /*posix stdout FD*/, msg.data(), msg.size());
+ // In production you'd want to check res, but error handling code will
+ // need to be fairly application specific if it's going to properly
+ // avoid reentrant calls to malloc.
+ (void)res;
+}
+
+// Signal handler to make a SIGINT call cancel rather than exit().
+static void sig_catch(int val) {
+ (void)val;
+ // This is avoiding the tricky bits of signal handling, notably that the
+ // underlying string manipulation and IO functions used by the the logger
+ // are unlikely to be reentrant.
+ //
+ // Production code could mask out all logging on handler entry and enable
+ // it again on exit; here we just assume it's "good enough" and some
+ // (possibly broken) log messages are better than none.
+
+ sighandler_direct_stdout(catch_enter);
+ if(fs) {
+ // This will invoke the callback immediately with an OperationCanceled status
+ fs->CancelPendingConnect();
+ sighandler_direct_stdout(catch_cancel);
+ }
+ sighandler_direct_stdout(catch_exit);
+}
+
+
+int main(int arg_token_count, const char **args) {
+ (void)args;
+ if(arg_token_count != 1) {
+ std::cerr << "usage: ./connect_cancel";
+ google::protobuf::ShutdownProtobufLibrary();
+ exit(EXIT_FAILURE);
+ }
+
+ // Register signal handle to asynchronously invoke cancel from outside the main thread.
+ signal(SIGINT, sig_catch);
+
+ // Generic setup/config code much like the other examples.
+ hdfs::Options options;
+ //Setting the config path to the default: "$HADOOP_CONF_DIR" or "/etc/hadoop/conf"
+ hdfs::ConfigurationLoader loader;
+ //Loading default config files core-site.xml and hdfs-site.xml from the config path
+ hdfs::optional<hdfs::HdfsConfiguration> config = loader.LoadDefaultResources<hdfs::HdfsConfiguration>();
+ //TODO: HDFS-9539 - after this is resolved, valid config will always be returned.
+ if(config){
+ //Loading options from the config
+ options = config->GetOptions();
+ }
+
+
+ // Start an IoService and some worker threads
+ std::shared_ptr<hdfs::IoService> service = hdfs::IoService::MakeShared();
+ if(nullptr == service) {
+ std::cerr << "Unable to create IoService" << std::endl;
+ fs.reset();
+ // Nasty hack to clean up for valgrind since we don't have the C++17 optional<T>::reset method
+ config = decltype(config)();
+ google::protobuf::ShutdownProtobufLibrary();
+ exit(EXIT_FAILURE);
+ }
+
+ unsigned int worker_count = service->InitDefaultWorkers();
+ if(worker_count < 1) {
+ std::cerr << "Unable to create IoService worker threads";
+ fs.reset();
+ service->Stop();
+ config = decltype(config)();
+ google::protobuf::ShutdownProtobufLibrary();
+ exit(EXIT_FAILURE);
+ }
+
+ // Set up and connect to the FileSystem
+ fs.reset(hdfs::FileSystem::New(service, "", options));
+ if(nullptr == fs) {
+ std::cerr << "Unable to create FileSystem" << std::endl;
+ fs.reset();
+ service->Stop();
+ config = decltype(config)();
+ google::protobuf::ShutdownProtobufLibrary();
+ exit(EXIT_FAILURE);
+ }
+
+ hdfs::Status status = fs->ConnectToDefaultFs();
+ if (!status.ok()) {
+ if(!options.defaultFS.get_host().empty()){
+ std::cerr << "Error connecting to " << options.defaultFS << ". " << status.ToString() << std::endl;
+ } else {
+ std::cerr << "Error connecting to the cluster: defaultFS is empty. " << status.ToString() << std::endl;
+ }
+ fs.reset();
+ service->Stop();
+ config = decltype(config)();
+ google::protobuf::ShutdownProtobufLibrary();
+ exit(EXIT_FAILURE);
+ }
+
+ fs.reset();
+ service->Stop();
+ config = decltype(config)();
+ google::protobuf::ShutdownProtobufLibrary();
+
+ return 0;
+}
+
+
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8783461e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
index 72434e6..2b15cf4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
@@ -299,6 +299,7 @@ int hdfsPreAttachFileMonitor(libhdfspp_file_event_callback handler, int64_t cook
* objects; NULL on error or empty result.
* errno is set to non-zero on error or zero on success.
**/
+LIBHDFS_EXTERNAL
hdfsFileInfo * hdfsFind(hdfsFS fs, const char* path, const char* name, uint32_t * numEntries);
@@ -314,6 +315,7 @@ hdfsFileInfo * hdfsFind(hdfsFS fs, const char* path, const char* name, uint32_t
* @param name Name to be given to the created snapshot (may be NULL)
* @return 0 on success, corresponding errno on failure
**/
+LIBHDFS_EXTERNAL
int hdfsCreateSnapshot(hdfsFS fs, const char* path, const char* name);
/**
@@ -324,6 +326,7 @@ int hdfsCreateSnapshot(hdfsFS fs, const char* path, const char* name);
* @param name Name of the snapshot to be deleted (must be non-blank)
* @return 0 on success, corresponding errno on failure
**/
+LIBHDFS_EXTERNAL
int hdfsDeleteSnapshot(hdfsFS fs, const char* path, const char* name);
/**
@@ -333,6 +336,7 @@ int hdfsDeleteSnapshot(hdfsFS fs, const char* path, const char* name);
* @param path Path to the directory to be made snapshottable (must be non-blank)
* @return 0 on success, corresponding errno on failure
**/
+LIBHDFS_EXTERNAL
int hdfsAllowSnapshot(hdfsFS fs, const char* path);
/**
@@ -342,8 +346,35 @@ int hdfsAllowSnapshot(hdfsFS fs, const char* path);
* @param path Path to the directory to be made non-snapshottable (must be non-blank)
* @return 0 on success, corresponding errno on failure
**/
+LIBHDFS_EXTERNAL
int hdfsDisallowSnapshot(hdfsFS fs, const char* path);
+/**
+ * Create a FileSystem based on the builder but don't connect
+ * @param bld Used to populate config options in the same manner as hdfsBuilderConnect.
+ * Does not free builder.
+ **/
+LIBHDFS_EXTERNAL
+hdfsFS hdfsAllocateFileSystem(struct hdfsBuilder *bld);
+
+/**
+ * Connect a FileSystem created with hdfsAllocateFileSystem
+ * @param fs A disconnected FS created with hdfsAllocateFileSystem
+ * @param bld The same or exact copy of the builder used for Allocate, we still need a few fields.
+ * Does not free builder.
+ * @return 0 on success, corresponding errno on failure
+ **/
+LIBHDFS_EXTERNAL
+int hdfsConnectAllocated(hdfsFS fs, struct hdfsBuilder *bld);
+
+/**
+ * Cancel a pending connection on a FileSystem
+ * @param fs A fs in the process of connecting using hdfsConnectAllocated in another thread.
+ * @return 0 on success, corresponding errno on failure
+ **/
+LIBHDFS_EXTERNAL
+int hdfsCancelPendingConnection(hdfsFS fs);
+
#ifdef __cplusplus
} /* end extern "C" */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8783461e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
index 5fbd3d8..46a1e61 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
@@ -217,6 +217,12 @@ class FileSystem {
virtual Status ConnectToDefaultFs() = 0;
/**
+ * Cancels any attempts to connect to the HDFS cluster.
+ * FileSystem is expected to be destroyed after invoking this.
+ */
+ virtual bool CancelPendingConnect() = 0;
+
+ /**
* Open a file on HDFS. The call issues an RPC to the NameNode to
* gather the locations of all blocks in the file and to return a
* new instance of the @ref InputStream object.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8783461e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
index dd7d00c..0608244 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
@@ -332,6 +332,94 @@ hdfsFS doHdfsConnect(optional<std::string> nn, optional<tPort> port, optional<st
}
}
+hdfsFS hdfsAllocateFileSystem(struct hdfsBuilder *bld) {
+ // Same idea as the first half of doHdfsConnect, but return the wrapped FS before
+ // connecting.
+ try {
+ errno = 0;
+ std::shared_ptr<IoService> io_service = IoService::MakeShared();
+
+ int io_thread_count = bld->config.GetOptions().io_threads_;
+ if(io_thread_count < 1) {
+ io_service->InitDefaultWorkers();
+ } else {
+ io_service->InitWorkers(io_thread_count);
+ }
+
+ FileSystem *fs = FileSystem::New(io_service, bld->user.value_or(""), bld->config.GetOptions());
+ if (!fs) {
+ ReportError(ENODEV, "Could not create FileSystem object");
+ return nullptr;
+ }
+
+ if (fsEventCallback) {
+ fs->SetFsEventCallback(fsEventCallback.value());
+ }
+
+ return new hdfs_internal(fs);
+ } catch (const std::exception &e) {
+ ReportException(e);
+ return nullptr;
+ } catch (...) {
+ ReportCaughtNonException();
+ return nullptr;
+ }
+ return nullptr;
+}
+
+int hdfsConnectAllocated(hdfsFS fs, struct hdfsBuilder *bld) {
+ if(!CheckSystem(fs)) {
+ return ENODEV;
+ }
+
+ if(!bld) {
+ ReportError(ENODEV, "No hdfsBuilder object supplied");
+ return ENODEV;
+ }
+
+ // Get C++ FS to do connect
+ FileSystem *fsImpl = fs->get_impl();
+ if(!fsImpl) {
+ ReportError(ENODEV, "Null FileSystem implementation");
+ return ENODEV;
+ }
+
+ // Unpack the required bits of the hdfsBuilder
+ optional<std::string> nn = bld->overrideHost;
+ optional<tPort> port = bld->overridePort;
+ optional<std::string> user = bld->user;
+
+ // try-catch in case some of the third-party stuff throws
+ try {
+ Status status;
+ if (nn || port) {
+ if (!port) {
+ port = kDefaultPort;
+ }
+ std::string port_as_string = std::to_string(*port);
+ status = fsImpl->Connect(nn.value_or(""), port_as_string);
+ } else {
+ status = fsImpl->ConnectToDefaultFs();
+ }
+
+ if (!status.ok()) {
+ Error(status);
+ return ENODEV;
+ }
+
+ // 0 to indicate a good connection
+ return 0;
+ } catch (const std::exception & e) {
+ ReportException(e);
+ return ENODEV;
+ } catch (...) {
+ ReportCaughtNonException();
+ return ENODEV;
+ }
+
+ return 0;
+}
+
hdfsFS hdfsConnect(const char *nn, tPort port) {
return hdfsConnectAsUser(nn, port, "");
}
@@ -350,6 +438,26 @@ hdfsFS hdfsConnectNewInstance(const char* nn, tPort port) {
return hdfsConnectAsUser(nn, port, "");
}
+int hdfsCancelPendingConnection(hdfsFS fs) {
+ // todo: stick an enum in hdfs_internal to check the connect state
+ if(!CheckSystem(fs)) {
+ return ENODEV;
+ }
+
+ FileSystem *fsImpl = fs->get_impl();
+ if(!fsImpl) {
+ ReportError(ENODEV, "Null FileSystem implementation");
+ return ENODEV;
+ }
+
+ bool canceled = fsImpl->CancelPendingConnect();
+ if(canceled) {
+ return 0;
+ } else {
+ return EINTR;
+ }
+}
+
int hdfsDisconnect(hdfsFS fs) {
try
{
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8783461e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/continuation/asio.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/continuation/asio.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/continuation/asio.h
index 3f650ce..193358f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/continuation/asio.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/continuation/asio.h
@@ -34,24 +34,6 @@ namespace asio_continuation {
using namespace continuation;
-template <class Stream, class MutableBufferSequence>
-class ReadContinuation : public Continuation {
-public:
- ReadContinuation(std::shared_ptr<Stream>& stream, const MutableBufferSequence &buffer)
- : stream_(stream), buffer_(buffer) {}
- virtual void Run(const Next &next) override {
- auto handler =
- [next](const asio::error_code &ec, size_t) { next(ToStatus(ec)); };
- asio::async_read(*stream_, buffer_, handler);
- }
-
-private:
- // prevent construction from raw ptr
- ReadContinuation(Stream *stream, MutableBufferSequence &buffer);
- std::shared_ptr<Stream> stream_;
- MutableBufferSequence buffer_;
-};
-
template <class Stream, class ConstBufferSequence>
class WriteContinuation : public Continuation {
public:
@@ -71,80 +53,12 @@ private:
ConstBufferSequence buffer_;
};
-template <class Socket, class Iterator>
-class ConnectContinuation : public Continuation {
-public:
- ConnectContinuation(Socket *socket, Iterator begin, Iterator end,
- Iterator *connected_endpoint)
- : socket_(socket), begin_(begin), end_(end),
- connected_endpoint_(connected_endpoint) {}
-
- virtual void Run(const Next &next) override {
- auto handler = [this, next](const asio::error_code &ec, Iterator it) {
- if (connected_endpoint_) {
- *connected_endpoint_ = it;
- }
- next(ToStatus(ec));
- };
- asio::async_connect(*socket_, begin_, end_, handler);
- }
-
-private:
- Socket *socket_;
- Iterator begin_;
- Iterator end_;
- Iterator *connected_endpoint_;
-};
-
-template <class OutputIterator>
-class ResolveContinuation : public Continuation {
-public:
- ResolveContinuation(::asio::io_service *io_service, const std::string &server,
- const std::string &service, OutputIterator result)
- : resolver_(*io_service), query_(server, service), result_(result) {}
-
- virtual void Run(const Next &next) override {
- using resolver = ::asio::ip::tcp::resolver;
- auto handler =
- [this, next](const asio::error_code &ec, resolver::iterator it) {
- if (!ec) {
- std::copy(it, resolver::iterator(), result_);
- }
- next(ToStatus(ec));
- };
- resolver_.async_resolve(query_, handler);
- }
-
-private:
- ::asio::ip::tcp::resolver resolver_;
- ::asio::ip::tcp::resolver::query query_;
- OutputIterator result_;
-};
-
template <class Stream, class ConstBufferSequence>
static inline Continuation *Write(std::shared_ptr<Stream> stream,
const ConstBufferSequence &buffer) {
return new WriteContinuation<Stream, ConstBufferSequence>(stream, buffer);
}
-template <class Stream, class MutableBufferSequence>
-static inline Continuation *Read(std::shared_ptr<Stream> stream,
- const MutableBufferSequence &buffer) {
- return new ReadContinuation<Stream, MutableBufferSequence>(stream, buffer);
-}
-
-template <class Socket, class Iterator>
-static inline Continuation *Connect(Socket *socket, Iterator begin,
- Iterator end) {
- return new ConnectContinuation<Socket, Iterator>(socket, begin, end, nullptr);
-}
-
-template <class OutputIterator>
-static inline Continuation *
-Resolve(::asio::io_service *io_service, const std::string &server,
- const std::string &service, OutputIterator result) {
- return new ResolveContinuation<OutputIterator>(io_service, server, service, result);
-}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8783461e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/continuation/continuation.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/continuation/continuation.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/continuation/continuation.h
index 4c9b8ad..18aa146 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/continuation/continuation.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/continuation/continuation.h
@@ -131,18 +131,6 @@ template <class State> inline void Pipeline<State>::Run(UserHandler &&handler) {
Schedule(Status::OK());
}
-template <class Handler> class BindContinuation : public Continuation {
-public:
- BindContinuation(const Handler &handler) : handler_(handler) {}
- virtual void Run(const Next &next) override { handler_(next); }
-
-private:
- Handler handler_;
-};
-
-template <class Handler> static inline Continuation *Bind(const Handler &handler) {
- return new BindContinuation<Handler>(handler);
-}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8783461e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc
index 6778bad..ab87506 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc
@@ -71,6 +71,17 @@ std::vector<std::string> SplitOnComma(const std::string &s, bool include_empty_s
return res;
}
+std::string RemoveSpaces(const std::string &str) {
+ std::string res;
+ for(unsigned int i=0; i<str.size(); i++) {
+ char curr = str[i];
+ if(curr != ' ') {
+ res += curr;
+ }
+ }
+ return res;
+}
+
// Prepend hdfs:// to string if there isn't already a scheme
// Converts unset optional into empty string
std::string PrependHdfsScheme(optional<std::string> str) {
@@ -92,6 +103,8 @@ struct ha_parse_error : public std::exception {
};
std::vector<NamenodeInfo> HdfsConfiguration::LookupNameService(const std::string &nameservice) {
+ LOG_TRACE(kRPC, << "HDFSConfiguration@" << this << "::LookupNameService( nameservice=" << nameservice<< " ) called");
+
std::vector<NamenodeInfo> namenodes;
try {
// Find namenodes that belong to nameservice
@@ -104,8 +117,10 @@ std::vector<NamenodeInfo> HdfsConfiguration::LookupNameService(const std::string
else
throw ha_parse_error("unable to find " + service_nodes);
- for(auto it=namenode_ids.begin(); it != namenode_ids.end(); it++)
- LOG_INFO(kRPC, << "Namenode: " << *it);
+ for(unsigned int i=0; i<namenode_ids.size(); i++) {
+ namenode_ids[i] = RemoveSpaces(namenode_ids[i]);
+ LOG_INFO(kRPC, << "Namenode: " << namenode_ids[i]);
+ }
}
// should this error if we only find 1 NN?
@@ -123,7 +138,11 @@ std::vector<NamenodeInfo> HdfsConfiguration::LookupNameService(const std::string
}
URI uri = node_uri.value();
- LOG_INFO(kRPC, << "Read the following HA Namenode URI from config" << uri.GetDebugString());
+ if(uri.str() == "") {
+ LOG_WARN(kRPC, << "Attempted to read info for nameservice " << nameservice << " node " << dom_node_name << " but didn't find anything.")
+ } else {
+ LOG_INFO(kRPC, << "Read the following HA Namenode URI from config" << uri.GetDebugString());
+ }
NamenodeInfo node(nameservice, *node_id, uri);
namenodes.push_back(node);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8783461e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc
index bc38be7..bd43091 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc
@@ -18,7 +18,7 @@
#include "namenode_info.h"
-#include "common/continuation/asio.h"
+#include "common/util.h"
#include "common/logging.h"
#include <sstream>
@@ -71,62 +71,107 @@ bool ResolveInPlace(::asio::io_service *ioservice, ResolvedNamenodeInfo &info) {
return true;
}
+typedef std::vector<asio::ip::tcp::endpoint> endpoint_vector;
+
+// RAII wrapper
+class ScopedResolver {
+ private:
+ ::asio::io_service *io_service_;
+ std::string host_;
+ std::string port_;
+ ::asio::ip::tcp::resolver::query query_;
+ ::asio::ip::tcp::resolver resolver_;
+ endpoint_vector endpoints_;
+
+ // Caller blocks on access if resolution isn't finished
+ std::shared_ptr<std::promise<Status>> result_status_;
+ public:
+ ScopedResolver(::asio::io_service *service, const std::string &host, const std::string &port) :
+ io_service_(service), host_(host), port_(port), query_(host, port), resolver_(*io_service_)
+ {
+ if(!io_service_)
+ LOG_ERROR(kAsyncRuntime, << "ScopedResolver@" << this << " passed nullptr to io_service");
+ }
-std::vector<ResolvedNamenodeInfo> BulkResolve(::asio::io_service *ioservice, const std::vector<NamenodeInfo> &nodes) {
- using namespace asio_continuation;
+ ~ScopedResolver() {
+ resolver_.cancel();
+ }
- typedef std::vector<asio::ip::tcp::endpoint> endpoint_vector;
- typedef Pipeline<endpoint_vector> resolve_pipeline_t;
+ bool BeginAsyncResolve() {
+ // result_status_ would only exist if this was previously called. Invalid state.
+ if(result_status_) {
+ LOG_ERROR(kAsyncRuntime, << "ScopedResolver@" << this << "::BeginAsyncResolve invalid call: may only be called once per instance");
+ return false;
+ } else if(!io_service_) {
+ LOG_ERROR(kAsyncRuntime, << "ScopedResolver@" << this << "::BeginAsyncResolve invalid call: null io_service");
+ return false;
+ }
+ // Now set up the promise, set it in async_resolve's callback
+ result_status_ = std::make_shared<std::promise<Status>>();
+
+ // Callback to pull a copy of endpoints out of resolver and set promise
+ auto callback = [this](const asio::error_code &ec, ::asio::ip::tcp::resolver::iterator out) {
+ if(!ec) {
+ std::copy(out, ::asio::ip::tcp::resolver::iterator(), std::back_inserter(endpoints_));
+ }
+ result_status_->set_value( ToStatus(ec) );
+ };
+ resolver_.async_resolve(query_, callback);
+ return true;
+ }
- std::vector<std::pair<resolve_pipeline_t*, std::shared_ptr<std::promise<Status>>>> pipelines;
- pipelines.reserve(nodes.size());
+ Status Join() {
+ if(!result_status_) {
+ std::ostringstream errmsg;
+ errmsg << "ScopedResolver@" << this << "Join invalid call: promise never set";
+ return Status::InvalidArgument(errmsg.str().c_str());
+ }
+
+ std::future<Status> future_result = result_status_->get_future();
+ Status res = future_result.get();
+ return res;
+ }
+
+ endpoint_vector GetEndpoints() {
+ // Explicitly return by value to decouple lifecycles.
+ return endpoints_;
+ }
+};
+
+std::vector<ResolvedNamenodeInfo> BulkResolve(::asio::io_service *ioservice, const std::vector<NamenodeInfo> &nodes) {
+ std::vector< std::unique_ptr<ScopedResolver> > resolvers;
+ resolvers.reserve(nodes.size());
std::vector<ResolvedNamenodeInfo> resolved_info;
- // This must never reallocate once async ops begin
resolved_info.reserve(nodes.size());
for(unsigned int i=0; i<nodes.size(); i++) {
std::string host = nodes[i].get_host();
std::string port = nodes[i].get_port();
- ResolvedNamenodeInfo resolved;
- resolved = nodes[i];
- resolved_info.push_back(resolved);
-
- // build the pipeline
- resolve_pipeline_t *pipeline = resolve_pipeline_t::Create();
- auto resolve_step = Resolve(ioservice, host, port, std::back_inserter(pipeline->state()));
- pipeline->Push(resolve_step);
-
- // make a status associated with current pipeline
- std::shared_ptr<std::promise<Status>> active_stat = std::make_shared<std::promise<Status>>();
- pipelines.push_back(std::make_pair(pipeline, active_stat));
-
- pipeline->Run([i,active_stat, &resolved_info](const Status &s, const endpoint_vector &ends){
- resolved_info[i].endpoints = ends;
- active_stat->set_value(s);
- });
-
+ resolvers.emplace_back(new ScopedResolver(ioservice, host, port));
+ resolvers[i]->BeginAsyncResolve();
}
// Join all async operations
- std::vector<ResolvedNamenodeInfo> return_set;
- for(unsigned int i=0; i<pipelines.size();i++) {
- std::shared_ptr<std::promise<Status>> promise = pipelines[i].second;
-
- std::future<Status> future = promise->get_future();
- Status stat = future.get();
-
- // Clear endpoints if we hit an error
- if(!stat.ok()) {
- LOG_WARN(kRPC, << "Unable to resolve endpoints for " << nodes[i].uri.str());
- resolved_info[i].endpoints.clear();
+ for(unsigned int i=0; i < resolvers.size(); i++) {
+ Status asyncReturnStatus = resolvers[i]->Join();
+
+ ResolvedNamenodeInfo info;
+ info = nodes[i];
+
+ if(asyncReturnStatus.ok()) {
+ // Copy out endpoints if things went well
+ info.endpoints = resolvers[i]->GetEndpoints();
+ } else {
+ LOG_ERROR(kAsyncRuntime, << "Unabled to resolve endpoints for host: " << nodes[i].get_host()
+ << " port: " << nodes[i].get_port());
}
- }
+ resolved_info.push_back(info);
+ }
return resolved_info;
}
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8783461e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.h
index 7f0e572..09d8188 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.h
@@ -19,6 +19,7 @@
#define LIB_COMMON_UTIL_H_
#include "hdfspp/status.h"
+#include "common/logging.h"
#include <sstream>
#include <mutex>
@@ -112,6 +113,73 @@ inline asio::ip::tcp::socket *get_asio_socket_ptr<asio::ip::tcp::socket>
//Check if the high bit is set
bool IsHighBitSet(uint64_t num);
+
+
+// Provide a way to do an atomic swap on a callback.
+// SetCallback, AtomicSwapCallback, and GetCallback can only be called once each.
+// AtomicSwapCallback and GetCallback must only be called after SetCallback.
+//
+// We can't throw on error, and since the callback is templated it's tricky to
+// generate generic dummy callbacks. Complain loudly in the log and get good
+// test coverage. It shouldn't be too hard to avoid invalid states.
+template <typename CallbackType>
+class SwappableCallbackHolder {
+ private:
+ std::mutex state_lock_;
+ CallbackType callback_;
+ bool callback_set_ = false;
+ bool callback_swapped_ = false;
+ bool callback_accessed_ = false;
+ public:
+ bool IsCallbackSet() {
+ mutex_guard swap_lock(state_lock_);
+ return callback_set_;
+ }
+
+ bool IsCallbackAccessed() {
+ mutex_guard swap_lock(state_lock_);
+ return callback_accessed_;
+ }
+
+ bool SetCallback(const CallbackType& callback) {
+ mutex_guard swap_lock(state_lock_);
+ if(callback_set_ || callback_swapped_ || callback_accessed_) {
+ LOG_ERROR(kAsyncRuntime, << "SetCallback violates access invariants.")
+ return false;
+ }
+ callback_ = callback;
+ callback_set_ = true;
+ return true;
+ }
+
+ CallbackType AtomicSwapCallback(const CallbackType& replacement, bool& swapped) {
+ mutex_guard swap_lock(state_lock_);
+ if(!callback_set_ || callback_swapped_) {
+ LOG_ERROR(kAsyncRuntime, << "AtomicSwapCallback violates access invariants.")
+ swapped = false;
+ } else if (callback_accessed_) {
+ // Common case where callback has been invoked but caller may not know
+ LOG_DEBUG(kAsyncRuntime, << "AtomicSwapCallback called after callback has been accessed");
+ return false;
+ }
+
+ CallbackType old = callback_;
+ callback_ = replacement;
+ callback_swapped_ = true;
+ swapped = true;
+ return old;
+ }
+ CallbackType GetCallback() {
+ mutex_guard swap_lock(state_lock_);
+ if(!callback_set_ || callback_accessed_) {
+ LOG_ERROR(kAsyncRuntime, << "GetCallback violates access invariants.")
+ }
+ callback_accessed_ = true;
+ return callback_;
+ }
+};
+
+
}
#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8783461e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
index a5f3aad..b46102a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
@@ -202,6 +202,7 @@ void FileSystemImpl::Connect(const std::string &server,
LOG_INFO(kFileSystem, << "FileSystemImpl::Connect(" << FMT_THIS_ADDR
<< ", server=" << server << ", service="
<< service << ") called");
+ connect_callback_.SetCallback(handler);
/* IoService::New can return nullptr */
if (!io_service_) {
@@ -236,8 +237,8 @@ void FileSystemImpl::Connect(const std::string &server,
}
- nn_.Connect(cluster_name_, /*server, service*/ resolved_namenodes, [this, handler](const Status & s) {
- handler(s, this);
+ nn_.Connect(cluster_name_, /*server, service*/ resolved_namenodes, [this](const Status & s) {
+ connect_callback_.GetCallback()(s, this);
});
}
@@ -286,6 +287,43 @@ int FileSystemImpl::WorkerThreadCount() {
}
}
+bool FileSystemImpl::CancelPendingConnect() {
+ if(!connect_callback_.IsCallbackSet()) {
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl@" << this << "::CancelPendingConnect called before Connect started");
+ return false;
+ }
+ if(connect_callback_.IsCallbackAccessed()) {
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl@" << this << "::CancelPendingConnect called after Connect completed");
+ return false;
+ }
+
+ // First invoke callback, then do proper teardown in RpcEngine and RpcConnection
+ ConnectCallback noop_callback = [](const Status &stat, FileSystem *fs) {
+ LOG_DEBUG(kFileSystem, << "Dummy callback invoked for canceled FileSystem@" << fs << "::Connect with status: " << stat.ToString());
+ };
+
+ bool callback_swapped = false;
+ ConnectCallback original_callback = connect_callback_.AtomicSwapCallback(noop_callback, callback_swapped);
+
+ if(callback_swapped) {
+ // Take original callback and invoke it as if it was canceled.
+ LOG_DEBUG(kFileSystem, << "Swapped in dummy callback. Invoking connect callback with canceled status.");
+ std::function<void(void)> wrapped_callback = [original_callback, this](){
+ // handling code expected to check status before dereferenceing 'this'
+ original_callback(Status::Canceled(), this);
+ };
+ io_service_->PostTask(wrapped_callback);
+ } else {
+ LOG_INFO(kFileSystem, << "Unable to cancel FileSystem::Connect. It hasn't been invoked yet or may have already completed.")
+ return false;
+ }
+
+ // Now push cancel down to clean up where possible and make sure the RpcEngine
+ // won't try to do retries in the background. The rest of the memory cleanup
+ // happens when this FileSystem is deleted by the user.
+ return nn_.CancelPendingConnect();
+}
+
void FileSystemImpl::Open(
const std::string &path,
const std::function<void(const Status &, FileHandle *)> &handler) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8783461e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
index 80978cf..fbc3967 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
@@ -46,6 +46,8 @@ namespace hdfs {
class FileSystemImpl : public FileSystem {
public:
MEMCHECKED_CLASS(FileSystemImpl)
+ typedef std::function<void(const Status &, FileSystem *)> ConnectCallback;
+
explicit FileSystemImpl(IoService *&io_service, const std::string& user_name, const Options &options);
explicit FileSystemImpl(std::shared_ptr<IoService>, const std::string& user_name, const Options &options);
~FileSystemImpl() override;
@@ -61,6 +63,9 @@ public:
const std::function<void(const Status &, FileSystem *)> &handler) override;
virtual Status ConnectToDefaultFs() override;
+ /* Cancel connection if FS is in the middle of one */
+ virtual bool CancelPendingConnect() override;
+
virtual void Open(const std::string &path,
const std::function<void(const Status &, FileHandle *)>
&handler) override;
@@ -197,6 +202,9 @@ private:
NameNodeOperations nn_;
std::shared_ptr<BadDataNodeTracker> bad_node_tracker_;
+ // Keep connect callback around in case it needs to be canceled
+ SwappableCallbackHolder<ConnectCallback> connect_callback_;
+
/**
* Runtime event monitoring handlers.
* Note: This is really handy to have for advanced usage but
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8783461e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
index 89acac3..9e2d90a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
@@ -45,6 +45,10 @@ void NameNodeOperations::Connect(const std::string &cluster_name,
engine_.Connect(cluster_name, servers, handler);
}
+bool NameNodeOperations::CancelPendingConnect() {
+ return engine_.CancelPendingConnect();
+}
+
void NameNodeOperations::GetBlockLocations(const std::string & path, uint64_t offset, uint64_t length,
std::function<void(const Status &, std::shared_ptr<const struct FileInfo>)> handler)
{
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8783461e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
index 60efacc..59b3512 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
@@ -52,6 +52,8 @@ public:
const std::vector<ResolvedNamenodeInfo> &servers,
std::function<void(const Status &)> &&handler);
+ bool CancelPendingConnect();
+
void GetBlockLocations(const std::string & path, uint64_t offset, uint64_t length,
std::function<void(const Status &, std::shared_ptr<const struct FileInfo>)> handler);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8783461e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
index 7c280b8..651225a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
@@ -150,7 +150,8 @@ RpcEngine::RpcEngine(::asio::io_service *io_service, const Options &options,
protocol_version_(protocol_version),
call_id_(0),
retry_timer(*io_service),
- event_handlers_(std::make_shared<LibhdfsEvents>())
+ event_handlers_(std::make_shared<LibhdfsEvents>()),
+ connect_canceled_(false)
{
LOG_DEBUG(kRPC, << "RpcEngine::RpcEngine called");
@@ -182,6 +183,16 @@ void RpcEngine::Connect(const std::string &cluster_name,
conn_->Connect(last_endpoints_, auth_info_, handler);
}
+bool RpcEngine::CancelPendingConnect() {
+ if(connect_canceled_) {
+ LOG_DEBUG(kRPC, << "RpcEngine@" << this << "::CancelPendingConnect called more than once");
+ return false;
+ }
+
+ connect_canceled_ = true;
+ return true;
+}
+
void RpcEngine::Shutdown() {
LOG_DEBUG(kRPC, << "RpcEngine::Shutdown called");
io_service_->post([this]() {
@@ -250,6 +261,14 @@ void RpcEngine::AsyncRpc(
LOG_TRACE(kRPC, << "RpcEngine::AsyncRpc called");
+ // In case user-side code isn't checking the status of Connect before doing RPC
+ if(connect_canceled_) {
+ io_service_->post(
+ [handler](){ handler(Status::Canceled()); }
+ );
+ return;
+ }
+
if (!conn_) {
conn_ = InitializeConnection();
conn_->ConnectAndFlush(last_endpoints_);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8783461e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
index 9191ab2..b4aef00 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
@@ -353,6 +353,8 @@ class RpcEngine : public LockFreeRpcEngine {
const std::vector<ResolvedNamenodeInfo> servers,
RpcCallback &handler);
+ bool CancelPendingConnect();
+
void AsyncRpc(const std::string &method_name,
const ::google::protobuf::MessageLite *req,
const std::shared_ptr<::google::protobuf::MessageLite> &resp,
@@ -418,6 +420,9 @@ private:
std::mutex engine_state_lock_;
+ // Once Connect has been canceled there is no going back
+ bool connect_canceled_;
+
// Keep endpoint info for all HA connections, a non-null ptr indicates
// that HA info was found in the configuation.
std::unique_ptr<HANamenodeTracker> ha_persisted_info_;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/52] [abbrv] hadoop git commit: HDFS-11436: libhdfs++: Fix race
condition in ScopedResolver. Contributed by James Clampffer.
Posted by jh...@apache.org.
HDFS-11436: libhdfs++: Fix race condition in ScopedResolver. Contributed by James Clampffer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/606d20c6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/606d20c6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/606d20c6
Branch: refs/heads/HDFS-8707
Commit: 606d20c6c0678be0cc7d0163defceacd1ffe03ae
Parents: 8783461
Author: James <jh...@apache.org>
Authored: Wed Feb 22 12:12:24 2017 -0500
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../src/main/native/libhdfspp/lib/common/namenode_info.cc | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/606d20c6/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc
index bd43091..bd3f44f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc
@@ -109,13 +109,14 @@ class ScopedResolver {
// Now set up the promise, set it in async_resolve's callback
result_status_ = std::make_shared<std::promise<Status>>();
+ std::shared_ptr<std::promise<Status>> shared_result = result_status_;
// Callback to pull a copy of endpoints out of resolver and set promise
- auto callback = [this](const asio::error_code &ec, ::asio::ip::tcp::resolver::iterator out) {
+ auto callback = [this, shared_result](const asio::error_code &ec, ::asio::ip::tcp::resolver::iterator out) {
if(!ec) {
std::copy(out, ::asio::ip::tcp::resolver::iterator(), std::back_inserter(endpoints_));
}
- result_status_->set_value( ToStatus(ec) );
+ shared_result->set_value( ToStatus(ec) );
};
resolver_.async_resolve(query_, callback);
return true;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/52] [abbrv] hadoop git commit: HDFS-11099: libhdfs++: Expose rack
id in hdfsDNInfo. Contributed by Xiaowei Zhu.
Posted by jh...@apache.org.
HDFS-11099: libhdfs++: Expose rack id in hdfsDNInfo. Contributed by Xiaowei Zhu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2524afbc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2524afbc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2524afbc
Branch: refs/heads/HDFS-8707
Commit: 2524afbc206d8f6f0344ee7de29e36b6c1b46e4e
Parents: b7d9a0f
Author: James <jh...@apache.org>
Authored: Mon Nov 7 10:41:59 2016 -0500
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../main/native/libhdfspp/include/hdfspp/block_location.h | 9 +++++++++
.../src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h | 1 +
.../src/main/native/libhdfspp/lib/bindings/c/hdfs.cc | 5 +++++
.../src/main/native/libhdfspp/lib/fs/filesystem.cc | 2 ++
.../src/main/native/libhdfspp/tests/hdfs_ext_test.cc | 1 +
5 files changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2524afbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/block_location.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/block_location.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/block_location.h
index cbe34be..5a03f41 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/block_location.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/block_location.h
@@ -41,6 +41,14 @@ public:
this->ip_addr = ip_addr;
}
+ std::string getNetworkLocation() const {
+ return network_location;
+ }
+
+ void setNetworkLocation(const std::string & location) {
+ this->network_location = location;
+ }
+
int getXferPort() const {
return xfer_port;
}
@@ -75,6 +83,7 @@ public:
private:
std::string hostname;
std::string ip_addr;
+ std::string network_location;
int xfer_port;
int info_port;
int IPC_port;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2524afbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
index b41857c..72434e6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
@@ -132,6 +132,7 @@ int hdfsBuilderConfGetLong(struct hdfsBuilder *bld, const char *key, int64_t *va
struct hdfsDNInfo {
const char * ip_address;
const char * hostname;
+ const char * network_location;
int xfer_port;
int info_port;
int IPC_port;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2524afbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
index a43d94f..dd7d00c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
@@ -1248,6 +1248,10 @@ int hdfsGetBlockLocations(hdfsFS fs, const char *path, struct hdfsBlockLocations
buf = new char[ppDNInfo.getIPAddr().size() + 1];
strncpy(buf, ppDNInfo.getIPAddr().c_str(), ppDNInfo.getIPAddr().size() + 1);
dn_info->ip_address = buf;
+
+ buf = new char[ppDNInfo.getNetworkLocation().size() + 1];
+ strncpy(buf, ppDNInfo.getNetworkLocation().c_str(), ppDNInfo.getNetworkLocation().size() + 1);
+ dn_info->network_location = buf;
}
}
@@ -1270,6 +1274,7 @@ int hdfsFreeBlockLocations(struct hdfsBlockLocations * blockLocations) {
auto location = &block->locations[j];
delete[] location->hostname;
delete[] location->ip_address;
+ delete[] location->network_location;
}
}
delete[] blockLocations->blocks;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2524afbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
index 72b30e5..5d5b9f2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
@@ -354,6 +354,8 @@ BlockLocation LocatedBlockToBlockLocation(const hadoop::hdfs::LocatedBlockProto
newInfo.setIPCPort(id.ipcport());
if (id.has_infosecureport())
newInfo.setInfoSecurePort(id.infosecureport());
+ if (datanode_info.has_location())
+ newInfo.setNetworkLocation(datanode_info.location());
dn_info.push_back(newInfo);
}
result.setDataNodes(dn_info);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2524afbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc
index c160f7f..400b97c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc
@@ -56,6 +56,7 @@ TEST_F(HdfsExtTest, TestGetBlockLocations) {
EXPECT_EQ(1, blocks->blocks->num_locations);
EXPECT_NE(nullptr, blocks->blocks->locations->hostname);
EXPECT_NE(nullptr, blocks->blocks->locations->ip_address);
+ EXPECT_NE(nullptr, blocks->blocks->locations->network_location);
EXPECT_NE(0, blocks->blocks->locations->xfer_port);
result = hdfsFreeBlockLocations(blocks);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/52] [abbrv] hadoop git commit: HDFS-10761: libhdfs++: Fix broken
logic in HA retry policy. Contributed by James Clampffer
Posted by jh...@apache.org.
HDFS-10761: libhdfs++: Fix broken logic in HA retry policy. Contributed by James Clampffer
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/05ddb310
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/05ddb310
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/05ddb310
Branch: refs/heads/HDFS-8707
Commit: 05ddb3108176a32cfda945ba4cb736c91b38877b
Parents: b9cf0e9
Author: James <jh...@apache.org>
Authored: Mon Aug 22 17:34:59 2016 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../native/libhdfspp/lib/common/retry_policy.cc | 11 +++--
.../main/native/libhdfspp/lib/rpc/rpc_engine.cc | 48 +++++++++-----------
2 files changed, 27 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/05ddb310/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.cc
index a885d53..ef78aca 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.cc
@@ -56,20 +56,21 @@ RetryAction FixedDelayWithFailover::ShouldRetry(const Status &s, uint64_t retrie
(void)isIdempotentOrAtMostOnce;
LOG_TRACE(kRPC, << "FixedDelayWithFailover::ShouldRetry(retries=" << retries << ", failovers=" << failovers << ")");
- if(s.code() == ::asio::error::timed_out && failovers < max_failover_retries_) {
+ if(failovers < max_failover_retries_ && (s.code() == ::asio::error::timed_out || s.get_server_exception_type() == Status::kStandbyException) )
+ {
// Try connecting to another NN in case this one keeps timing out
// Can add the backoff wait specified by dfs.client.failover.sleep.base.millis here
return RetryAction::failover(delay_);
}
- if(retries < max_retries_) {
- LOG_TRACE(kRPC, << "FixedDelayWithFailover::ShouldRetry: retries < max_retries_");
+ if(retries < max_retries_ && failovers < max_failover_retries_) {
+ LOG_TRACE(kRPC, << "FixedDelayWithFailover::ShouldRetry: retries < max_retries_ && failovers < max_failover_retries_");
return RetryAction::retry(delay_);
} else if (retries >= max_retries_ && failovers < max_failover_retries_) {
LOG_TRACE(kRPC, << "FixedDelayWithFailover::ShouldRetry: retries >= max_retries_ && failovers < max_failover_retries_");
return RetryAction::failover(delay_);
- } else if (retries >= max_retries_ && failovers == max_failover_retries_) {
- LOG_TRACE(kRPC, << "FixedDelayWithFailover::ShouldRetry: retries >= max_retries_ && failovers == max_failover_retries_");
+ } else if (retries <= max_retries_ && failovers == max_failover_retries_) {
+ LOG_TRACE(kRPC, << "FixedDelayWithFailover::ShouldRetry: retries <= max_retries_ && failovers == max_failover_retries_");
// 1 last retry on new connection
return RetryAction::retry(delay_);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/05ddb310/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
index be69d95..89d6a67 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
@@ -291,39 +291,33 @@ void RpcEngine::RpcCommsError(
optional<RetryAction> head_action = optional<RetryAction>();
- //We are talking to the Standby NN, let's talk to the active one instead.
- if(ha_persisted_info_ && status.get_server_exception_type() == Status::kStandbyException) {
- LOG_INFO(kRPC, << "Received StandbyException. Failing over.");
- head_action = RetryAction::failover(std::max(0,options_.rpc_retry_delay_ms));
- } else {
- // Filter out anything with too many retries already
- for (auto it = pendingRequests.begin(); it < pendingRequests.end();) {
- auto req = *it;
+ // Filter out anything with too many retries already
+ for (auto it = pendingRequests.begin(); it < pendingRequests.end();) {
+ auto req = *it;
- LOG_DEBUG(kRPC, << req->GetDebugString());
+ LOG_DEBUG(kRPC, << req->GetDebugString());
- RetryAction retry = RetryAction::fail(""); // Default to fail
+ RetryAction retry = RetryAction::fail(""); // Default to fail
- if (retry_policy()) {
- retry = retry_policy()->ShouldRetry(status, req->IncrementRetryCount(), req->get_failover_count(), true);
- }
+ if (retry_policy()) {
+ retry = retry_policy()->ShouldRetry(status, req->IncrementRetryCount(), req->get_failover_count(), true);
+ }
- if (retry.action == RetryAction::FAIL) {
- // If we've exceeded the maximum retry, take the latest error and pass it
- // on. There might be a good argument for caching the first error
- // rather than the last one, that gets messy
+ if (retry.action == RetryAction::FAIL) {
+ // If we've exceeded the maximum retry, take the latest error and pass it
+ // on. There might be a good argument for caching the first error
+ // rather than the last one, that gets messy
- io_service().post([req, status]() {
- req->OnResponseArrived(nullptr, status); // Never call back while holding a lock
- });
- it = pendingRequests.erase(it);
- } else {
- if (!head_action) {
- head_action = retry;
- }
-
- ++it;
+ io_service().post([req, status]() {
+ req->OnResponseArrived(nullptr, status); // Never call back while holding a lock
+ });
+ it = pendingRequests.erase(it);
+ } else {
+ if (!head_action) {
+ head_action = retry;
}
+
+ ++it;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/52] [abbrv] hadoop git commit: HDFS-11758: libhdfs++: Catch
exceptions thrown by runtime hooks. Contributed by James Clampffer.
Posted by jh...@apache.org.
HDFS-11758: libhdfs++: Catch exceptions thrown by runtime hooks. Contributed by James Clampffer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c2386bc2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c2386bc2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c2386bc2
Branch: refs/heads/HDFS-8707
Commit: c2386bc2f33d0bec984d31bc5f599c622732e751
Parents: 57cdad7
Author: James Clampffer <ja...@hp.com>
Authored: Tue May 16 15:29:51 2017 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../native/libhdfspp/include/hdfspp/events.h | 82 +++++----
.../native/libhdfspp/lib/bindings/c/hdfs.cc | 8 +-
.../libhdfspp/lib/common/libhdfs_events_impl.cc | 38 +++++
.../libhdfspp/lib/common/libhdfs_events_impl.h | 29 +---
.../main/native/libhdfspp/lib/fs/filehandle.cc | 4 +-
.../native/libhdfspp/lib/reader/block_reader.cc | 12 +-
.../libhdfspp/lib/rpc/rpc_connection_impl.cc | 2 +-
.../libhdfspp/lib/rpc/rpc_connection_impl.h | 4 +-
.../native/libhdfspp/tests/bad_datanode_test.cc | 2 +-
.../native/libhdfspp/tests/hdfs_ext_test.cc | 168 ++++++++++++++++++-
.../native/libhdfspp/tests/rpc_engine_test.cc | 2 +-
11 files changed, 281 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2386bc2/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/events.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/events.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/events.h
index 80c3712..496703a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/events.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/events.h
@@ -51,24 +51,54 @@ static constexpr const char * FILE_DN_WRITE_EVENT = "DN::write";
class event_response {
public:
-// Create a response
-enum event_response_type {
- kOk = 0,
+ // Helper factories
+ // The default ok response; libhdfspp should continue normally
+ static event_response make_ok() {
+ return event_response(kOk);
+ }
+ static event_response make_caught_std_exception(const char *what) {
+ return event_response(kCaughtStdException, what);
+ }
+ static event_response make_caught_unknown_exception() {
+ return event_response(kCaughtUnknownException);
+ }
- // Responses to be used in testing only
- kTest_Error = 100
-};
+ // High level classification of responses
+ enum event_response_type {
+ kOk = 0,
+ // User supplied callback threw.
+ // Std exceptions will copy the what() string
+ kCaughtStdException = 1,
+ kCaughtUnknownException = 2,
+ // Responses to be used in testing only
+ kTest_Error = 100
+ };
- // The default ok response; libhdfspp should continue normally
- static event_response ok() { return event_response(); }
- event_response_type response() { return response_; }
+ event_response_type response_type() { return response_type_; }
private:
- event_response() : response_(event_response_type::kOk) {};
+ // Use factories to construct for now
+ event_response();
+ event_response(event_response_type type)
+ : response_type_(type)
+ {
+ if(type == kCaughtUnknownException) {
+ status_ = Status::Exception("c++ unknown exception", "");
+ }
+ }
+ event_response(event_response_type type, const char *what)
+ : response_type_(type),
+ exception_msg_(what==nullptr ? "" : what)
+ {
+ status_ = Status::Exception("c++ std::exception", exception_msg_.c_str());
+ }
+
- event_response_type response_;
+ event_response_type response_type_;
+ // use to hold what str if event handler threw
+ std::string exception_msg_;
///////////////////////////////////////////////
@@ -83,31 +113,23 @@ public:
return event_response(status);
}
- Status status() { return error_status_; }
+ Status status() { return status_; }
private:
event_response(const Status & status) :
- response_(event_response_type::kTest_Error), error_status_(status) {}
+ response_type_(event_response_type::kTest_Error), status_(status) {}
- Status error_status_; // To be used with kTest_Error
+ Status status_; // To be used with kTest_Error
};
-
-
/* callback signature */
-typedef std::function<
- event_response (const char * event,
- const char * cluster,
- int64_t value)>
- fs_event_callback;
-
-typedef std::function<
- event_response (const char * event,
- const char * cluster,
- const char * file,
- int64_t value)>
- file_event_callback;
-
-
+typedef std::function<event_response (const char * event,
+ const char * cluster,
+ int64_t value)> fs_event_callback;
+
+typedef std::function<event_response (const char * event,
+ const char * cluster,
+ const char * file,
+ int64_t value)>file_event_callback;
}
#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2386bc2/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
index 2856b0d..4e6c0f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
@@ -1462,7 +1462,7 @@ event_response fs_callback_glue(libhdfspp_fs_event_callback handler,
int64_t value) {
int result = handler(event, cluster, value, cookie);
if (result == LIBHDFSPP_EVENT_OK) {
- return event_response::ok();
+ return event_response::make_ok();
}
#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
if (result == DEBUG_SIMULATE_ERROR) {
@@ -1470,7 +1470,7 @@ event_response fs_callback_glue(libhdfspp_fs_event_callback handler,
}
#endif
- return event_response::ok();
+ return event_response::make_ok();
}
event_response file_callback_glue(libhdfspp_file_event_callback handler,
@@ -1481,7 +1481,7 @@ event_response file_callback_glue(libhdfspp_file_event_callback handler,
int64_t value) {
int result = handler(event, cluster, file, value, cookie);
if (result == LIBHDFSPP_EVENT_OK) {
- return event_response::ok();
+ return event_response::make_ok();
}
#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
if (result == DEBUG_SIMULATE_ERROR) {
@@ -1489,7 +1489,7 @@ event_response file_callback_glue(libhdfspp_file_event_callback handler,
}
#endif
- return event_response::ok();
+ return event_response::make_ok();
}
HDFS_EXT_API
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2386bc2/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/libhdfs_events_impl.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/libhdfs_events_impl.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/libhdfs_events_impl.cc
index bcf9ccc..d9f4edf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/libhdfs_events_impl.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/libhdfs_events_impl.cc
@@ -18,6 +18,8 @@
#include "libhdfs_events_impl.h"
+#include <exception>
+
namespace hdfs {
/**
@@ -46,6 +48,42 @@ void LibhdfsEvents::clear_file_callback() {
file_callback = std::experimental::nullopt;
}
+event_response LibhdfsEvents::call(const char * event,
+ const char * cluster,
+ int64_t value)
+{
+ if (fs_callback) {
+ try {
+ return fs_callback->operator()(event, cluster, value);
+ } catch (const std::exception& e) {
+ return event_response::make_caught_std_exception(e.what());
+ } catch (...) {
+ // Arguably calling abort() here would serve as appropriate
+ // punishment for those who throw garbage that isn't derived
+ // from std::exception...
+ return event_response::make_caught_unknown_exception();
+ }
+ } else {
+ return event_response::make_ok();
+ }
+}
+event_response LibhdfsEvents::call(const char * event,
+ const char * cluster,
+ const char * file,
+ int64_t value)
+{
+ if (file_callback) {
+ try {
+ return file_callback->operator()(event, cluster, file, value);
+ } catch (const std::exception& e) {
+ return event_response::make_caught_std_exception(e.what());
+ } catch (...) {
+ return event_response::make_caught_unknown_exception();
+ }
+ } else {
+ return event_response::make_ok();
+ }
+}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2386bc2/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/libhdfs_events_impl.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/libhdfs_events_impl.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/libhdfs_events_impl.h
index 122f7b0..73b20fb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/libhdfs_events_impl.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/libhdfs_events_impl.h
@@ -40,27 +40,14 @@ public:
void clear_fs_callback();
void clear_file_callback();
- event_response call(const char * event,
- const char * cluster,
- int64_t value) {
- if (fs_callback) {
- return fs_callback->operator ()(event, cluster, value);
- } else {
- return event_response::ok();
- }
- }
-
- event_response call(const char * event,
- const char * cluster,
- const char * file,
- int64_t value) {
- if (file_callback) {
- return file_callback->operator ()(event, cluster, file, value);
- } else {
- return event_response::ok();
- }
- }
-
+ event_response call(const char *event,
+ const char *cluster,
+ int64_t value);
+
+ event_response call(const char *event,
+ const char *cluster,
+ const char *file,
+ int64_t value);
private:
// Called when fs events occur
std::experimental::optional<fs_event_callback> fs_callback;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2386bc2/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
index 24e81b6..eea7ac9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
@@ -256,7 +256,7 @@ void FileHandleImpl::AsyncPreadSome(
auto read_handler = [reader, event_handlers, cluster_name, path, dn_id, handler](const Status & status, size_t transferred) {
event_response event_resp = event_handlers->call(FILE_DN_READ_EVENT, cluster_name.c_str(), path.c_str(), transferred);
#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
- if (event_resp.response() == event_response::kTest_Error) {
+ if (event_resp.response_type() == event_response::kTest_Error) {
handler(event_resp.status(), dn_id, transferred);
return;
}
@@ -270,7 +270,7 @@ void FileHandleImpl::AsyncPreadSome(
(void)dn;
event_response event_resp = event_handlers->call(FILE_DN_CONNECT_EVENT, cluster_name.c_str(), path.c_str(), 0);
#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
- if (event_resp.response() == event_response::kTest_Error) {
+ if (event_resp.response_type() == event_response::kTest_Error) {
status = event_resp.status();
}
#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2386bc2/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc
index b63ef76..877911c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc
@@ -122,7 +122,7 @@ void BlockReaderImpl::AsyncRequestBlock(const std::string &client_name,
if(this->event_handlers_) {
event_response event_resp = this->event_handlers_->call(FILE_DN_READ_EVENT, "", "", 0);
#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
- if (stat.ok() && event_resp.response() == event_response::kTest_Error) {
+ if (stat.ok() && event_resp.response_type() == event_response::kTest_Error) {
stat = Status::Error("Test error");
}
#endif
@@ -182,7 +182,7 @@ struct BlockReaderImpl::ReadPacketHeader : continuation::Continuation
if(parent_->event_handlers_) {
event_response event_resp = parent_->event_handlers_->call(FILE_DN_READ_EVENT, "", "", 0);
#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
- if (status.ok() && event_resp.response() == event_response::kTest_Error) {
+ if (status.ok() && event_resp.response_type() == event_response::kTest_Error) {
status = Status::Error("Test error");
}
#endif
@@ -255,7 +255,7 @@ struct BlockReaderImpl::ReadChecksum : continuation::Continuation
if(parent->event_handlers_) {
event_response event_resp = parent->event_handlers_->call(FILE_DN_READ_EVENT, "", "", 0);
#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
- if (status.ok() && event_resp.response() == event_response::kTest_Error) {
+ if (status.ok() && event_resp.response_type() == event_response::kTest_Error) {
status = Status::Error("Test error");
}
#endif
@@ -309,7 +309,7 @@ struct BlockReaderImpl::ReadData : continuation::Continuation
if(parent_->event_handlers_) {
event_response event_resp = parent_->event_handlers_->call(FILE_DN_READ_EVENT, "", "", 0);
#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
- if (status.ok() && event_resp.response() == event_response::kTest_Error) {
+ if (status.ok() && event_resp.response_type() == event_response::kTest_Error) {
status = Status::Error("Test error");
}
#endif
@@ -360,7 +360,7 @@ struct BlockReaderImpl::ReadPadding : continuation::Continuation
if(parent_->event_handlers_) {
event_response event_resp = parent_->event_handlers_->call(FILE_DN_READ_EVENT, "", "", 0);
#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
- if (status.ok() && event_resp.response() == event_response::kTest_Error) {
+ if (status.ok() && event_resp.response_type() == event_response::kTest_Error) {
status = Status::Error("Test error");
}
#endif
@@ -414,7 +414,7 @@ struct BlockReaderImpl::AckRead : continuation::Continuation
if(parent_->event_handlers_) {
event_response event_resp = parent_->event_handlers_->call(FILE_DN_READ_EVENT, "", "", 0);
#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
- if (status.ok() && event_resp.response() == event_response::kTest_Error) {
+ if (status.ok() && event_resp.response_type() == event_response::kTest_Error) {
status = Status::Error("Test error");
}
#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2386bc2/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
index 1012a37..198bcdc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
@@ -189,7 +189,7 @@ Status RpcConnection::HandleRpcResponse(std::shared_ptr<Response> response) {
if(event_handlers_) {
event_response event_resp = event_handlers_->call(FS_NN_READ_EVENT, cluster_name_.c_str(), 0);
#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
- if (event_resp.response() == event_response::kTest_Error) {
+ if (event_resp.response_type() == event_response::kTest_Error) {
status = event_resp.status();
}
#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2386bc2/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h
index 12f7f0e..4339a48 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h
@@ -179,7 +179,7 @@ void RpcConnectionImpl<Socket>::ConnectComplete(const ::asio::error_code &ec, co
if(event_handlers_) {
event_response event_resp = event_handlers_->call(FS_NN_CONNECT_EVENT, cluster_name_.c_str(), 0);
#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
- if (event_resp.response() == event_response::kTest_Error) {
+ if (event_resp.response_type() == event_response::kTest_Error) {
status = event_resp.status();
}
#endif
@@ -370,7 +370,7 @@ void RpcConnectionImpl<Socket>::OnRecvCompleted(const ::asio::error_code &origin
if(event_handlers_) {
event_response event_resp = event_handlers_->call(FS_NN_READ_EVENT, cluster_name_.c_str(), 0);
#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
- if (event_resp.response() == event_response::kTest_Error) {
+ if (event_resp.response_type() == event_response::kTest_Error) {
my_ec = std::make_error_code(std::errc::network_down);
}
#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2386bc2/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc
index 5185484..00bbf3d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc
@@ -189,7 +189,7 @@ TEST(BadDataNodeTest, NNEventCallback) {
if (calls++ == 1)
return event_response::test_err(Status::Error("Test"));
- return event_response::ok();
+ return event_response::make_ok();
});
PartialMockFileHandle is("cluster", "file", &io_service.io_service(), GetRandomClientName(), file_info, tracker, monitors);
Status stat;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2386bc2/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc
index 400b97c..7d94944 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc
@@ -18,7 +18,10 @@
#include "hdfspp_mini_dfs.h"
#include "hdfspp/hdfs_ext.h"
+
+#include <cstring>
#include <chrono>
+#include <exception>
namespace hdfs {
@@ -324,8 +327,8 @@ TEST_F(HdfsExtTest, TestEOF) {
EXPECT_EQ(0, hdfsCloseFile(fs, file));
}
- //Testing hdfsExists
- TEST_F(HdfsExtTest, TestExists) {
+//Testing hdfsExists
+TEST_F(HdfsExtTest, TestExists) {
HdfsHandle connection = cluster.connect_c();
hdfsFS fs = connection.handle();
@@ -585,8 +588,169 @@ TEST_F(HdfsExtTest, TestWorkingDirectory) {
}
+// Flags used to test event handlers
+static int connect_callback_invoked = 0;
+int basic_fs_callback(const char *event, const char *cluster, int64_t value, int64_t cookie) {
+ (void)cluster;
+ (void)value;
+ if(::strstr(FS_NN_CONNECT_EVENT, event) && cookie == 0xFFF0) {
+ connect_callback_invoked = 1;
+ }
+ return LIBHDFSPP_EVENT_OK;
+}
+
+// Make sure event handler gets called during connect
+TEST_F(HdfsExtTest, TestConnectEvent) {
+ connect_callback_invoked = 0;
+ hdfsPreAttachFSMonitor(basic_fs_callback, 0xFFF0);
+
+ HdfsHandle connection = cluster.connect_c();
+ hdfsFS fs = connection.handle();
+ EXPECT_NE(nullptr, fs);
+ EXPECT_EQ(connect_callback_invoked, 1);
+}
+
+int throwing_fs_callback(const char *event, const char *cluster, int64_t value, int64_t cookie) {
+ (void)cluster;
+ (void)value;
+ if(::strstr(FS_NN_CONNECT_EVENT, event) && cookie == 0xFFF1) {
+ connect_callback_invoked = 1;
+ throw std::runtime_error("Throwing in callbacks is a bad thing.");
+ }
+ return LIBHDFSPP_EVENT_OK;
+}
+
+// Make sure throwing in the connect event handler doesn't prevent connection
+TEST_F(HdfsExtTest, TestConnectEventThrow) {
+ connect_callback_invoked = 0;
+ hdfsPreAttachFSMonitor(throwing_fs_callback, 0xFFF1);
+
+ HdfsHandle connection = cluster.connect_c();
+ hdfsFS fs = connection.handle();
+ EXPECT_NE(nullptr, fs);
+ EXPECT_EQ(connect_callback_invoked, 1);
+}
+
+int char_throwing_fs_callback(const char *event, const char *cluster, int64_t value, int64_t cookie) {
+ (void)cluster;
+ (void)value;
+ if(::strstr(FS_NN_CONNECT_EVENT, event) && cookie == 0xFFF2) {
+ connect_callback_invoked = 1;
+ throw "Throwing non std::exceptions in callbacks is even worse.";
+ }
+ return LIBHDFSPP_EVENT_OK;
+}
+
+TEST_F(HdfsExtTest, TestConnectEventThrowChar) {
+ connect_callback_invoked = 0;
+ hdfsPreAttachFSMonitor(char_throwing_fs_callback, 0xFFF2);
+
+ HdfsHandle connection = cluster.connect_c();
+ hdfsFS fs = connection.handle();
+ EXPECT_NE(nullptr, fs);
+ EXPECT_EQ(connect_callback_invoked, 1);
+}
+
+// Make sure throwing in the read event handler doesn't prevent reads
+int read_handler_invokation_count = 0;
+int basic_read_event_handler(const char *event, const char *cluster, const char *file,
+ int64_t value, int64_t cookie)
+{
+ (void)cluster;
+ (void)file;
+ (void)value;
+ if(::strstr(FILE_DN_READ_EVENT, event) && cookie == 0xFFF3) {
+ read_handler_invokation_count += 1;
+ }
+ return LIBHDFSPP_EVENT_OK;
}
+// Testing that read handler is called.
+// Note: This is counting calls to async_read rather than hdfsPread.
+// Typically a call to hdfs(P)Read that doesn't span blocks/packets
+// invokes async_read 6 times; 4 more than required (improving that
+// in HDFS-11266).
+TEST_F(HdfsExtTest, TestReadEvent) {
+ read_handler_invokation_count = 0;
+ hdfsPreAttachFileMonitor(basic_read_event_handler, 0xFFF3);
+
+ HdfsHandle connection = cluster.connect_c();
+ hdfsFS fs = connection.handle();
+ EXPECT_NE(nullptr, fs);
+ //Write to a file
+ errno = 0;
+ int size = 256;
+ std::string path = "/readEventTest";
+ hdfsFile file = hdfsOpenFile(fs, path.c_str(), O_WRONLY, 0, 0, 0);
+ EXPECT_NE(nullptr, file);
+ void * buf = malloc(size);
+ memset(buf, ' ', size);
+ EXPECT_EQ(size, hdfsWrite(fs, file, buf, size));
+ free(buf);
+ EXPECT_EQ(0, hdfsCloseFile(fs, file));
+
+ //Test that read counters are getting incremented
+ char buffer[300];
+ file = hdfsOpenFile(fs, path.c_str(), O_RDONLY, 0, 0, 0);
+ EXPECT_EQ(size, hdfsPread(fs, file, 0, buffer, sizeof(buffer)));
+ EXPECT_EQ(read_handler_invokation_count, 6);
+
+ EXPECT_EQ(size, hdfsPread(fs, file, 0, buffer, sizeof(buffer)));
+ EXPECT_EQ(read_handler_invokation_count, 12);
+
+
+ EXPECT_EQ(0, hdfsCloseFile(fs, file));
+}
+
+int throwing_read_event_handler(const char *event, const char *cluster, const char *file,
+ int64_t value, int64_t cookie)
+{
+ (void)cluster;
+ (void)file;
+ (void)value;
+ if(::strstr(FILE_DN_READ_EVENT, event) && cookie == 0xFFF4) {
+ read_handler_invokation_count += 1;
+ throw std::runtime_error("Throwing here is a bad idea, but shouldn't break reads");
+ }
+ return LIBHDFSPP_EVENT_OK;
+}
+
+// Testing that reads can be done when event handler throws.
+TEST_F(HdfsExtTest, TestReadEventThrow) {
+ read_handler_invokation_count = 0;
+ hdfsPreAttachFileMonitor(throwing_read_event_handler, 0xFFF4);
+
+ HdfsHandle connection = cluster.connect_c();
+ hdfsFS fs = connection.handle();
+ EXPECT_NE(nullptr, fs);
+ //Write to a file
+ errno = 0;
+ int size = 256;
+ std::string path = "/readEventTest";
+ hdfsFile file = hdfsOpenFile(fs, path.c_str(), O_WRONLY, 0, 0, 0);
+ EXPECT_NE(nullptr, file);
+ void * buf = malloc(size);
+ memset(buf, ' ', size);
+ EXPECT_EQ(size, hdfsWrite(fs, file, buf, size));
+ free(buf);
+ EXPECT_EQ(0, hdfsCloseFile(fs, file));
+
+ //Test that read counters are getting incremented
+ char buffer[300];
+ file = hdfsOpenFile(fs, path.c_str(), O_RDONLY, 0, 0, 0);
+ EXPECT_EQ(size, hdfsPread(fs, file, 0, buffer, sizeof(buffer)));
+ EXPECT_EQ(read_handler_invokation_count, 6);
+
+ EXPECT_EQ(size, hdfsPread(fs, file, 0, buffer, sizeof(buffer)));
+ EXPECT_EQ(read_handler_invokation_count, 12);
+
+
+ EXPECT_EQ(0, hdfsCloseFile(fs, file));
+}
+
+
+} // end namespace hdfs
+
int main(int argc, char *argv[]) {
// The following line must be executed to initialize Google Mock
// (and Google Test) before running the tests.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2386bc2/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
index 66a5f1b..2b6cdbc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
@@ -367,7 +367,7 @@ TEST(RpcEngineTest, TestEventCallbacks)
if (calls == 1 || calls == 3) // First connect and first read
return event_response::test_err(Status::Error("Test"));
- return event_response::ok();
+ return event_response::make_ok();
});
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/52] [abbrv] hadoop git commit: HDFS-10874: libhdfs++: Public API
headers should not depend on internal implementation. Contributed by James
Clampffer
Posted by jh...@apache.org.
HDFS-10874: libhdfs++: Public API headers should not depend on internal implementation. Contributed by James Clampffer
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53e40bc7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53e40bc7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53e40bc7
Branch: refs/heads/HDFS-8707
Commit: 53e40bc7189598d42d92f208522f00a224c95719
Parents: 3db261d
Author: James Clampffer <ja...@hp.com>
Authored: Fri Jul 21 16:14:52 2017 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../native/libhdfspp/include/hdfspp/hdfspp.h | 13 +-
.../native/libhdfspp/include/hdfspp/options.h | 2 +-
.../main/native/libhdfspp/include/hdfspp/uri.h | 137 +++++++++++
.../libhdfspp/lib/common/configuration.cc | 20 +-
.../native/libhdfspp/lib/common/configuration.h | 2 +-
.../libhdfspp/lib/common/hdfs_configuration.cc | 7 +-
.../libhdfspp/lib/common/namenode_info.cc | 8 +-
.../main/native/libhdfspp/lib/common/options.cc | 11 +-
.../src/main/native/libhdfspp/lib/common/uri.cc | 164 ++++++++-----
.../src/main/native/libhdfspp/lib/common/uri.h | 130 ----------
.../main/native/libhdfspp/lib/fs/filesystem.cc | 13 +-
.../src/main/native/libhdfspp/tests/uri_test.cc | 239 +++++++++++--------
.../libhdfspp/tools/hdfs_allowSnapshot.cc | 10 +-
.../src/main/native/libhdfspp/tools/hdfs_cat.cc | 10 +-
.../main/native/libhdfspp/tools/hdfs_chgrp.cc | 12 +-
.../main/native/libhdfspp/tools/hdfs_chmod.cc | 12 +-
.../main/native/libhdfspp/tools/hdfs_chown.cc | 12 +-
.../native/libhdfspp/tools/hdfs_copyToLocal.cc | 10 +-
.../main/native/libhdfspp/tools/hdfs_count.cc | 10 +-
.../libhdfspp/tools/hdfs_createSnapshot.cc | 10 +-
.../libhdfspp/tools/hdfs_deleteSnapshot.cc | 10 +-
.../src/main/native/libhdfspp/tools/hdfs_df.cc | 8 +-
.../libhdfspp/tools/hdfs_disallowSnapshot.cc | 10 +-
.../src/main/native/libhdfspp/tools/hdfs_du.cc | 12 +-
.../main/native/libhdfspp/tools/hdfs_find.cc | 10 +-
.../src/main/native/libhdfspp/tools/hdfs_get.cc | 10 +-
.../src/main/native/libhdfspp/tools/hdfs_ls.cc | 12 +-
.../main/native/libhdfspp/tools/hdfs_mkdir.cc | 10 +-
.../native/libhdfspp/tools/hdfs_moveToLocal.cc | 10 +-
.../libhdfspp/tools/hdfs_renameSnapshot.cc | 10 +-
.../src/main/native/libhdfspp/tools/hdfs_rm.cc | 10 +-
.../main/native/libhdfspp/tools/hdfs_setrep.cc | 10 +-
.../main/native/libhdfspp/tools/hdfs_stat.cc | 10 +-
.../main/native/libhdfspp/tools/hdfs_tail.cc | 14 +-
.../main/native/libhdfspp/tools/tools_common.cc | 15 +-
.../main/native/libhdfspp/tools/tools_common.h | 3 +
36 files changed, 523 insertions(+), 473 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
index d193b5f..40ceef5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
@@ -25,6 +25,7 @@
#include "hdfspp/statinfo.h"
#include "hdfspp/fsinfo.h"
#include "hdfspp/content_summary.h"
+#include "hdfspp/uri.h"
#include <functional>
#include <memory>
@@ -199,9 +200,17 @@ class FileSystem {
**/
static FileSystem *New();
+ /**
+ * Callback type for async FileSystem::Connect calls.
+ * Provides the result status and instance pointer to the handler.
+ **/
+ typedef std::function<void(const Status& result_status, FileSystem *created_fs)> AsyncConnectCallback;
+ /**
+ * Connect directly to the specified namenode using the host and port (service).
+ **/
virtual void Connect(const std::string &server, const std::string &service,
- const std::function<void(const Status &, FileSystem *)> &handler) = 0;
+ const AsyncConnectCallback &handler) = 0;
/* Synchronous call of Connect */
virtual Status Connect(const std::string &server, const std::string &service) = 0;
@@ -214,7 +223,7 @@ class FileSystem {
* If no defaultFs is defined, returns an error.
*/
virtual void ConnectToDefaultFs(
- const std::function<void(const Status &, FileSystem *)> &handler) = 0;
+ const AsyncConnectCallback& handler) = 0;
virtual Status ConnectToDefaultFs() = 0;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
index 456390f..0a42314 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
@@ -18,7 +18,7 @@
#ifndef LIBHDFSPP_OPTIONS_H_
#define LIBHDFSPP_OPTIONS_H_
-#include "common/uri.h"
+#include "hdfspp/uri.h"
#include <string>
#include <vector>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/uri.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/uri.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/uri.h
new file mode 100644
index 0000000..d8574d1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/uri.h
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef COMMON_HDFS_URI_H_
+#define COMMON_HDFS_URI_H_
+
+#include <iostream>
+#include <string>
+#include <vector>
+#include <stdexcept>
+
+namespace hdfs
+{
+
+class uri_parse_error : public std::invalid_argument {
+ public:
+ uri_parse_error(const char *what_str) : std::invalid_argument(what_str) {}
+ uri_parse_error(const std::string& what_str) : std::invalid_argument(what_str) {}
+};
+
+class URI {
+public:
+ // Parse a string into a URI. Throw a hdfs::uri_parse_error if URI is malformed.
+ static URI parse_from_string(const std::string &str);
+
+ // URI encode/decode strings
+ static std::string encode (const std::string &input);
+ static std::string decode (const std::string &input);
+
+ URI();
+
+ std::string get_scheme(bool encoded_output=false) const;
+
+ void set_scheme(const std::string &s, bool encoded_input=false);
+
+ // empty if none.
+ std::string get_host(bool encoded_output=false) const;
+
+ void set_host(const std::string& h, bool encoded_input=false);
+
+ // true if port has been set
+ bool has_port() const;
+
+ // undefined if port hasn't been set
+ uint16_t get_port() const;
+
+ // use default if port hasn't been set
+ uint16_t get_port_or_default(uint16_t default_val) const;
+
+ void set_port(uint16_t p);
+
+ void clear_port();
+
+ std::string get_path(bool encoded_output=false) const;
+
+ void set_path(const std::string &p, bool encoded_input=false);
+
+ void add_path(const std::string &p, bool encoded_input=false);
+
+ std::vector<std::string> get_path_elements(bool encoded_output=false) const;
+
+ struct Query {
+ Query(const std::string& key, const std::string& val);
+ std::string key;
+ std::string value;
+ };
+
+ std::string get_query(bool encoded_output=false) const;
+
+ std::vector<Query> get_query_elements(bool encoded_output=false) const;
+
+ // Not that set_query must always pass in encoded strings
+ void set_query(const std::string &q);
+
+ // Adds a parameter onto the query; does not check if it already exists
+ // e.g. parseFromString("foo?bar=baz").addQuery("bing","bang")
+ // would leave "bar=baz&bing=bang" as the query
+ void add_query(const std::string &name, const std::string & value, bool encoded_input=false);
+
+ // Removes the query part if exists
+ // e.g. parseFromString("foo?bar=baz&bing=bang&bar=bong").removeQueries("bar")
+ // would leave bing=bang as the query
+ void remove_query(const std::string &q_name, bool encoded_input=false);
+
+ std::string get_fragment(bool encoded_output=false) const;
+
+ void set_fragment(const std::string &f, bool encoded_input=false);
+
+ std::string str(bool encoded_output=true) const;
+
+ // Get a string with each URI field printed on a seperate line
+ std::string GetDebugString() const;
+private:
+ // These are stored in encoded form
+ std::string scheme;
+ std::string user;
+ std::string pass;
+ std::string host;
+ std::vector<std::string> path;
+ std::vector<Query> queries;
+ std::string fragment;
+ // implicitly narrowed to uint16_t if positive
+ // -1 to indicate uninitialized
+ int32_t _port;
+
+ // URI encoding helpers
+ static std::string from_encoded(bool encoded_output, const std::string & input);
+ static std::string to_encoded(bool encoded_input, const std::string & input);
+
+ bool has_authority() const;
+ std::string build_authority(bool encoded_output) const;
+
+ std::string build_path(bool encoded_output) const;
+ void parse_path(bool input_encoded, const std::string &input_path);
+};
+
+inline std::ostream& operator<<(std::ostream &out, const URI &uri) {
+ return out << uri.str();
+}
+
+}
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.cc
index ac9a089..7fbf7c9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.cc
@@ -32,7 +32,7 @@
*/
#include "configuration.h"
-#include "uri.h"
+#include "hdfspp/uri.h"
#include <strings.h>
#include <sstream>
@@ -140,12 +140,15 @@ bool Configuration::GetBoolWithDefault(const std::string& key,
}
optional<URI> Configuration::GetUri(const std::string& key) const {
- auto raw = Get(key);
+ optional<std::string> raw = Get(key);
if (raw) {
- return URI::parse_from_string(*raw);
- } else {
- return optional<URI>();
+ try {
+ return std::experimental::make_optional(URI::parse_from_string(*raw));
+ } catch (const uri_parse_error& e) {
+ // Return empty below
+ }
}
+ return optional<URI>();
}
URI Configuration::GetUriWithDefault(const std::string& key,
@@ -154,10 +157,9 @@ URI Configuration::GetUriWithDefault(const std::string& key,
if (result) {
return *result;
} else {
- result = URI::parse_from_string(default_value);
- if (result) {
- return *result;
- } else {
+ try {
+ return URI::parse_from_string(default_value);
+ } catch (const uri_parse_error& e) {
return URI();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.h
index ebfa909..1e7ee5b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.h
@@ -19,7 +19,7 @@
#ifndef COMMON_CONFIGURATION_H_
#define COMMON_CONFIGURATION_H_
-#include "common/uri.h"
+#include "hdfspp/uri.h"
#include <string>
#include <map>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc
index ab87506..07e2edc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc
@@ -131,13 +131,14 @@ std::vector<NamenodeInfo> HdfsConfiguration::LookupNameService(const std::string
for(auto node_id=namenode_ids.begin(); node_id != namenode_ids.end(); node_id++) {
// find URI
std::string dom_node_name = std::string("dfs.namenode.rpc-address.") + nameservice + "." + *node_id;
- optional<URI> node_uri = URI::parse_from_string(PrependHdfsScheme(Get(dom_node_name)));
- if(!node_uri) {
+ URI uri;
+ try {
+ uri = URI::parse_from_string(PrependHdfsScheme(Get(dom_node_name)));
+ } catch (const uri_parse_error) {
throw ha_parse_error("unable to find " + dom_node_name);
}
- URI uri = node_uri.value();
if(uri.str() == "") {
LOG_WARN(kRPC, << "Attempted to read info for nameservice " << nameservice << " node " << dom_node_name << " but didn't find anything.")
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc
index bd3f44f..d29f1e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc
@@ -41,11 +41,11 @@ std::string ResolvedNamenodeInfo::str() const {
std::stringstream ss;
ss << "ResolvedNamenodeInfo {nameservice: " << nameservice << ", name: " << name << ", uri: " << uri.str();
ss << ", host: " << uri.get_host();
- auto port = uri.get_port();
- if(port)
- ss << ", port: " << port.value();
+
+ if(uri.has_port())
+ ss << ", port: " << uri.get_port();
else
- ss << ", port: unable to parse";
+ ss << ", invalid port (uninitialized)";
ss << ", scheme: " << uri.get_scheme();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc
index 728082e..48c83f1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc
@@ -48,11 +48,14 @@ Options::Options() : rpc_timeout(kDefaultRpcTimeout),
std::string NamenodeInfo::get_host() const {
return uri.get_host();
}
+
std::string NamenodeInfo::get_port() const {
- optional<uint16_t> p = uri.get_port();
- if(!p)
- return std::to_string(-1);
- return std::to_string(p.value());
+ if(uri.has_port()) {
+ return std::to_string(uri.get_port());
+ }
+ return "-1";
}
+
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.cc
index a54be8a..2213f8b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.cc
@@ -17,17 +17,16 @@
*/
-#include <common/uri.h>
+#include <hdfspp/uri.h>
#include <uriparser2/uriparser/Uri.h>
#include <string.h>
#include <sstream>
#include <cstdlib>
+#include <cassert>
#include <limits>
-using std::experimental::nullopt;
-
namespace hdfs
{
@@ -140,25 +139,23 @@ std::string copy_range(const UriTextRangeA *r) {
return "";
}
-bool parse_int(const UriTextRangeA *r, optional<uint16_t> * result) {
- assert(result); // output
- std::string int_string = copy_range(r);
- if (!int_string.empty()) {
+bool parse_int(const UriTextRangeA *r, int32_t& result)
+{
+ std::string int_str = copy_range(r);
+ if(!int_str.empty()) {
errno = 0;
- unsigned long val = ::strtoul(int_string.c_str(), nullptr, 10);
- if (errno == 0 && val < std::numeric_limits<uint16_t>::max() ) {
- *result = std::experimental::make_optional<uint16_t>(val);
+ unsigned long val = ::strtoul(int_str.c_str(), nullptr, 10);
+ if(errno == 0 && val < std::numeric_limits<uint16_t>::max()) {
+ result = val;
return true;
} else {
return false;
}
}
-
- // No value
- *result = nullopt;
return true;
}
+
std::vector<std::string> copy_path(const UriPathSegmentA *ps) {
std::vector<std::string> result;
if (nullptr == ps)
@@ -190,8 +187,8 @@ void parse_user_info(const UriTextRangeA *r, std::string * user, std::string * p
}
-std::vector<std::pair<std::string, std::string > > parse_query(const char *first, const char * afterLast) {
- std::vector<std::pair<std::string, std::string > > result;
+std::vector<URI::Query> parse_queries(const char *first, const char * afterLast) {
+ std::vector<URI::Query> result;
UriQueryListA * query;
int count;
int dissect_result = uriDissectQueryMallocExA(&query, &count, first, afterLast, false, URI_BR_DONT_TOUCH);
@@ -199,7 +196,7 @@ std::vector<std::pair<std::string, std::string > > parse_query(const char *first
for (auto ps = query; ps != nullptr; ps = ps->next) {
std::string key = ps->key ? URI::encode(ps->key) : "";
std::string value = ps->value ? URI::encode(ps->value) : "";
- result.push_back(std::make_pair(key, value));
+ result.emplace_back(key, value);
}
uriFreeQueryListA(query);
}
@@ -207,8 +204,8 @@ std::vector<std::pair<std::string, std::string > > parse_query(const char *first
return result;
}
-
-optional<URI> URI::parse_from_string(const std::string &str)
+// Parse a string into a URI. Throw a hdfs::uri_parse_error if URI is malformed.
+URI URI::parse_from_string(const std::string &str)
{
URI ret;
bool ok = true;
@@ -224,9 +221,9 @@ optional<URI> URI::parse_from_string(const std::string &str)
if (ok) {
ret.scheme = copy_range(&uu.scheme);
ret.host = copy_range(&uu.hostText);
- ok &= parse_int(&uu.portText, &ret.port);
+ ok &= parse_int(&uu.portText, ret._port);
ret.path = copy_path(uu.pathHead);
- ret.query = parse_query(uu.query.first, uu.query.afterLast);
+ ret.queries = parse_queries(uu.query.first, uu.query.afterLast);
ret.fragment = copy_range(&uu.fragment);
parse_user_info(&uu.userInfo, &ret.user, &ret.pass);
uriFreeUriMembersA(&uu);
@@ -234,9 +231,9 @@ optional<URI> URI::parse_from_string(const std::string &str)
uriFreeUriMembersA(&uu);
if (ok) {
- return std::experimental::make_optional(ret);
+ return ret;
} else {
- return nullopt;
+ throw uri_parse_error(str);
}
}
@@ -246,6 +243,9 @@ optional<URI> URI::parse_from_string(const std::string &str)
//
///////////////////////////////////////////////////////////////////////////////
+URI::URI() : _port(-1) {}
+URI::Query::Query(const std::string& k, const std::string& v) : key(k), value(v) {}
+
std::string URI::str(bool encoded_output) const
{
std::stringstream ss;
@@ -257,7 +257,7 @@ std::string URI::str(bool encoded_output) const
}
if (has_authority()) ss << build_authority(encoded_output);
if (!path.empty()) ss << get_path(encoded_output);
- if (!query.empty()) ss << "?" << get_query(encoded_output);
+ if (!queries.empty()) ss << "?" << get_query(encoded_output);
if (!fragment.empty()) ss << "#" << from_encoded(encoded_output, fragment);
return ss.str();
@@ -265,25 +265,62 @@ std::string URI::str(bool encoded_output) const
bool URI::has_authority() const
{
- return (!host.empty()) || (port);
+ return (!host.empty()) || (has_port());
}
std::string URI::build_authority(bool encoded_output) const
{
std::stringstream ss;
ss << URI::from_encoded(encoded_output, host);
- if (port)
+ if (has_port())
{
- ss << ":" << *port;
+ ss << ":" << _port;
}
return ss.str();
}
+std::string URI::get_scheme(bool encoded_output) const {
+ return from_encoded(encoded_output,scheme);
+}
+
+void URI::set_scheme(const std::string &s, bool encoded_input) {
+ scheme = to_encoded(encoded_input,s);
+}
+
+std::string URI::get_host(bool encoded_output) const {
+ return from_encoded(encoded_output,host);
+}
+
+void URI::set_host(const std::string& h, bool encoded_input) {
+ host = to_encoded(encoded_input,h);
+}
+
+bool URI::has_port() const {
+ return _port != -1;
+}
+
+uint16_t URI::get_port() const {
+ return (uint16_t)_port;
+}
+
+uint16_t URI::get_port_or_default(uint16_t val) const {
+ return has_port() ? (uint16_t)_port : val;
+}
+
+void URI::set_port(uint16_t p)
+{
+ _port = (int32_t)p & 0xFFFF;
+}
+
+void URI::clear_port()
+{
+ _port = -1;
+}
std::string URI::get_path(bool encoded_output) const
{
std::ostringstream out;
- for (auto s: path) {
+ for (const std::string& s: path) {
out << "/" << from_encoded(encoded_output, s);
}
return out.str();
@@ -292,7 +329,7 @@ std::string URI::get_path(bool encoded_output) const
std::vector<std::string> URI::get_path_elements(bool encoded_output) const
{
std::vector<std::string> result;
- for (auto path_elem: path) {
+ for (const std::string& path_elem: path) {
result.push_back(from_encoded(encoded_output, path_elem));
}
@@ -302,72 +339,89 @@ std::vector<std::string> URI::get_path_elements(bool encoded_output) const
void URI::parse_path(bool input_encoded, const std::string &input_path)
{
std::vector<std::string> split_path = split(input_path, '/');
- for (auto s: split_path) {
+ for (const std::string& s: split_path) {
path.push_back(to_encoded(input_encoded, s));
}
}
-
// Mostly copied and modified from uriparser2.c
+void URI::set_path(const std::string &p, bool encoded_input) {
+ parse_path(encoded_input, p);
+}
+
void URI::add_path(const std::string &p, bool encoded_input)
{
path.push_back(to_encoded(encoded_input, p));
}
-
std::string URI::get_query(bool encoded_output) const {
bool first = true;
std::stringstream ss;
- for (auto q: query) {
+ for (const Query& q: queries) {
if (!first) {
ss << "&";
}
- ss << from_encoded(encoded_output, q.first) << "=" << from_encoded(encoded_output, q.second);
+ ss << from_encoded(encoded_output, q.key) << "=" << from_encoded(encoded_output, q.value);
first = false;
}
return ss.str();
}
-std::vector< std::pair<std::string, std::string> > URI::get_query_elements(bool encoded_output) const
+std::vector<URI::Query> URI::get_query_elements(bool encoded_output) const
{
- std::vector< std::pair<std::string, std::string> > result;
- for (auto q: query) {
- auto key = from_encoded(encoded_output, q.first);
- auto value = from_encoded(encoded_output, q.second);
- result.push_back(std::make_pair(key, value));
+ std::vector<Query> result;
+ for (const Query& q: queries) {
+ std::string key = from_encoded(encoded_output, q.key);
+ std::string value = from_encoded(encoded_output, q.value);
+ result.emplace_back(key, value);
}
return result;
}
-
void URI::set_query(const std::string &q) {
- query = parse_query(q.c_str(), q.c_str() + q.size() + 1);
+ queries = parse_queries(q.c_str(), q.c_str() + q.size() + 1);
}
void URI::add_query(const std::string &name, const std::string & value, bool encoded_input)
{
- query.push_back(std::make_pair(to_encoded(encoded_input, name), to_encoded(encoded_input, value)));
+ queries.emplace_back(to_encoded(encoded_input, name), to_encoded(encoded_input, value));
}
-void URI::remove_queries(const std::string &q_name, bool encoded_input)
+void URI::remove_query(const std::string &q_name, bool encoded_input)
{
- if (query.empty())
+ if (queries.empty())
return;
// This is the one place we need to do decoded comparisons
std::string decoded_key = encoded_input ? decode(q_name) : q_name;
- for (int i = query.size() - 1; i >= 0; i--) {
- if (decode(query[i].first) == decoded_key) {
- query.erase(query.begin() + i);
+ for (int i = queries.size() - 1; i >= 0; i--) {
+ if (decode(queries[i].key) == decoded_key) {
+ queries.erase(queries.begin() + i);
}
}
}
+std::string URI::get_fragment(bool encoded_output) const {
+ return from_encoded(encoded_output, fragment);
+}
+
+void URI::set_fragment(const std::string &f, bool encoded_input) {
+ fragment = to_encoded(encoded_input,f);
+}
+
+std::string URI::from_encoded(bool encoded_output, const std::string & input) {
+ return encoded_output ? input : decode(input);
+}
+
+std::string URI::to_encoded(bool encoded_input, const std::string & input) {
+ return encoded_input ? input : encode(input);
+}
+
std::string URI::GetDebugString() const {
std::stringstream ss;
ss << std::endl;
@@ -375,25 +429,25 @@ std::string URI::GetDebugString() const {
ss << "\t" << "uri.get_scheme() = \"" << get_scheme() << "\"" << std::endl;
ss << "\t" << "uri.get_host() = \"" << get_host() << "\"" << std::endl;
- if(!port)
- ss << "\t" << "uri.get_port() = unset optional<uint16_t>" << std::endl;
+ if(_port == -1)
+ ss << "\t" << "uri.get_port() = invalid (uninitialized)" << std::endl;
else
- ss << "\t" << "uri.get_port() = \"" << port.value() << "\"" << std::endl;
+ ss << "\t" << "uri.get_port() = \"" << _port << "\"" << std::endl;
ss << "\t" << "uri.get_path() = \"" << get_path() << "\"" << std::endl;
ss << "\t" << "uri.get_fragment() = \"" << get_fragment() << "\"" << std::endl;
- std::vector<std::pair<std::string, std::string> > elems = get_query_elements();
+ std::vector<Query> query_elems = get_query_elements();
- if(elems.size() > 0)
+ if(query_elems.size() > 0)
ss << "\t" << "Query elements:" << std::endl;
- for(auto qry = elems.begin(); qry != elems.end(); qry++) {
- ss << "\t\t" << qry->first << " -> " << qry->second << std::endl;
+ for(auto qry = query_elems.begin(); qry != query_elems.end(); qry++) {
+ ss << "\t\t" << qry->key << " -> " << qry->value << std::endl;
}
return ss.str();
}
-}
+} // end namespace hdfs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.h
deleted file mode 100644
index 87f6919..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.h
+++ /dev/null
@@ -1,130 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef COMMON_HDFS_URI_H_
-#define COMMON_HDFS_URI_H_
-
-#include <iostream>
-#include <string>
-#include <optional.hpp>
-#include <vector>
-
-namespace hdfs
-{
-
-template <class T>
-using optional = std::experimental::optional<T>;
-
-class URI
-{
- // These are stored in encoded form
- std::string scheme;
- std::string user;
- std::string pass;
- std::string host;
- optional<uint16_t> port;
- std::vector<std::string> path;
- std::vector<std::pair<std::string,std::string> > query;
- std::string fragment;
-
- template <class T>
- static T from_encoded(bool encoded_output, const T & input) {return encoded_output ? input : decode(input);}
-
- template <class T>
- static T to_encoded(bool encoded_input, const T & input) {return encoded_input ? input : encode(input);}
-
- bool has_authority() const;
- std::string build_authority(bool encoded_output) const;
-
- std::string build_path(bool encoded_output) const;
- void parse_path(bool input_encoded, const std::string &input_path);
-
-public:
- // Parse a string into a URI. Returns nullopt if the URI is malformed.
- static optional<URI> parse_from_string(const std::string &str);
-
- static std::string encode (const std::string &input);
- static std::string decode (const std::string &input);
-
- std::string get_scheme(bool encoded_output=false) const
- { return from_encoded(encoded_output,scheme); }
-
- void set_scheme(const std::string &s, bool encoded_input=false)
- { scheme = to_encoded(encoded_input,s); }
-
- // empty if none.
- std::string get_host(bool encoded_output=false) const
- { return from_encoded(encoded_output,host); }
-
- void set_host(const std::string& h, bool encoded_input=false)
- { host = to_encoded(encoded_input,h); }
-
- // -1 if the port is undefined.
- optional<uint16_t> get_port() const
- { return port; }
-
- void set_port(uint16_t p)
- { port = p; }
-
- void clear_port()
- { port = std::experimental::nullopt; }
-
- std::string get_path(bool encoded_output=false) const;
-
- std::vector<std::string> get_path_elements(bool encoded_output=false) const;
-
- void set_path(const std::string &p, bool encoded_input=false) {
- parse_path(encoded_input, p);
- }
-
- void add_path(const std::string &p, bool encoded_input=false);
-
- std::string get_query(bool encoded_output=false) const;
-
- std::vector< std::pair<std::string, std::string> > get_query_elements(bool encoded_output=false) const;
-
- // Not that set_query must always pass in encoded strings
- void set_query(const std::string &q);
-
- // Adds a parameter onto the query; does not check if it already exists
- // e.g. parseFromString("foo?bar=baz").addQuery("bing","bang")
- // would leave "bar=baz&bing=bang" as the query
- void add_query(const std::string &name, const std::string & value, bool encoded_input=false);
-
- // Removes the query part if exists
- // e.g. parseFromString("foo?bar=baz&bing=bang&bar=bong").removeQueries("bar")
- // would leave bing=bang as the query
- void remove_queries(const std::string &q_name, bool encoded_input=false);
-
- std::string get_fragment(bool encoded_output=false) const
- { return from_encoded(encoded_output, fragment); }
-
- void set_fragment(const std::string &f, bool encoded_input=false)
- { fragment = to_encoded(encoded_input,f); }
-
- std::string str(bool encoded_output=true) const;
-
- // Get a string with each URI field printed on a seperate line
- std::string GetDebugString() const;
-};
-
-inline std::ostream& operator<<(std::ostream &out, const URI &uri)
-{ return out << uri.str(); }
-
-}
-#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
index 761ff86..c340796 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
@@ -221,12 +221,12 @@ void FileSystemImpl::Connect(const std::string &server,
// tmp namenode info just to get this in the right format for BulkResolve
NamenodeInfo tmp_info;
- optional<URI> uri = URI::parse_from_string("hdfs://" + cluster_name_);
- if(!uri) {
+ try {
+ tmp_info.uri = URI::parse_from_string("hdfs://" + cluster_name_);
+ } catch (const uri_parse_error& e) {
LOG_ERROR(kFileSystem, << "Unable to use URI for cluster " << cluster_name_);
handler(Status::Error(("Invalid namenode " + cluster_name_ + " in config").c_str()), this);
}
- tmp_info.uri = uri.value();
resolved_namenodes = BulkResolve(&io_service_->io_service(), {tmp_info});
}
@@ -258,11 +258,8 @@ void FileSystemImpl::ConnectToDefaultFs(const std::function<void(const Status &,
return;
}
- optional<uint16_t> port = options_.defaultFS.get_port();
- if (!port) {
- port = kDefaultPort;
- }
- std::string port_as_string = std::to_string(*port);
+ int16_t port = options_.defaultFS.get_port_or_default(kDefaultPort);
+ std::string port_as_string = std::to_string(port);
Connect(host, port_as_string, handler);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/uri_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/uri_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/uri_test.cc
index 1dd870a..78f1a58 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/uri_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/uri_test.cc
@@ -16,103 +16,127 @@
* limitations under the License.
*/
-#include "common/uri.h"
+#include "hdfspp/uri.h"
#include <gmock/gmock.h>
using ::testing::_;
using namespace hdfs;
+
+URI expect_uri_throw(const char *uri) {
+ bool threw = false;
+ std::string what_msg;
+ URI val;
+ try {
+ val = URI::parse_from_string(uri);
+ } catch (const uri_parse_error& e) {
+ threw = true;
+ what_msg = e.what();
+ } catch (...) {
+ threw = true;
+ }
+
+ EXPECT_TRUE(threw);
+ EXPECT_EQ(what_msg, uri);
+ return val;
+}
+
+URI expect_uri_nothrow(const char *uri) {
+ bool threw = false;
+ std::string what_msg;
+ URI val;
+ try {
+ val = URI::parse_from_string(uri);
+ } catch (const uri_parse_error& e) {
+ threw = true;
+ what_msg = e.what();
+ } catch (...) {
+ threw = true;
+ }
+
+ EXPECT_FALSE(threw);
+ EXPECT_EQ(what_msg, "");
+ return val;
+}
+
+
TEST(UriTest, TestDegenerateInputs) {
/* Empty input */
- {
- optional<URI> uri = URI::parse_from_string("");
- EXPECT_TRUE(uri && "Empty input");
- }
+ expect_uri_nothrow("");
/* Invalid encoding */
- {
- optional<URI> uri = URI::parse_from_string("%%");
- EXPECT_FALSE(uri && "Bad input");
- }
+ expect_uri_throw("%%");
/* Invalid port */
- {
- optional<URI> uri = URI::parse_from_string("hdfs://nn:foo/");
- EXPECT_FALSE(uri && "Bad port");
- }
+ expect_uri_throw("hdfs://nn:foo/");
/* Negative port */
- {
- optional<URI> uri = URI::parse_from_string("hdfs://nn:-100/");
- EXPECT_FALSE(uri && "Negative port");
- }
+ expect_uri_throw("hdfs://nn:-100/");
/* Empty paths */
- {
- optional<URI> uri = URI::parse_from_string("hdfs://////");
- EXPECT_TRUE(uri && "Empty paths");
- }
-
+ expect_uri_nothrow("hdfs://////");
}
TEST(UriTest, TestNominalInputs) {
/* Simple input */
{
- optional<URI> uri = URI::parse_from_string("hdfs:///foo");
- ASSERT_TRUE(uri && "Parsed");
- EXPECT_EQ("hdfs", uri->get_scheme());
- EXPECT_EQ("", uri->get_host());
- EXPECT_EQ(0, uri->get_port().value_or(0));
- EXPECT_EQ("/foo", uri->get_path());
- EXPECT_EQ("", uri->get_fragment());
- EXPECT_EQ("", uri->get_query());
+ URI uri = expect_uri_nothrow("hdfs:///foo");
+ EXPECT_EQ("hdfs", uri.get_scheme());
+ EXPECT_EQ("", uri.get_host());
+ EXPECT_FALSE(uri.has_port());
+ EXPECT_EQ(0, uri.get_port_or_default(0));
+ EXPECT_EQ("/foo", uri.get_path());
+ EXPECT_EQ("", uri.get_fragment());
+ EXPECT_EQ("", uri.get_query());
}
/* With authority */
{
- optional<URI> uri = URI::parse_from_string("hdfs://host:100/foo");
- ASSERT_TRUE(uri && "Parsed");
- EXPECT_EQ("hdfs", uri->get_scheme());
- EXPECT_EQ("host", uri->get_host());
- EXPECT_EQ(100, uri->get_port().value_or(0));
- EXPECT_EQ("/foo", uri->get_path());
- EXPECT_EQ("", uri->get_fragment());
- EXPECT_EQ("", uri->get_query());
+ URI uri = expect_uri_nothrow("hdfs://host:100/foo");
+ EXPECT_EQ("hdfs", uri.get_scheme());
+ EXPECT_EQ("host", uri.get_host());
+ EXPECT_TRUE(uri.has_port());
+ EXPECT_EQ(100, uri.get_port());
+ EXPECT_EQ(100, uri.get_port_or_default(0));
+ EXPECT_EQ("/foo", uri.get_path());
+ EXPECT_EQ("", uri.get_fragment());
+ EXPECT_EQ("", uri.get_query());
}
/* No scheme */
{
- optional<URI> uri = URI::parse_from_string("/foo");
- ASSERT_TRUE(uri && "Parsed");
- EXPECT_EQ("", uri->get_scheme());
- EXPECT_EQ("", uri->get_host());
- EXPECT_EQ(0, uri->get_port().value_or(0));
- EXPECT_EQ("/foo", uri->get_path());
- EXPECT_EQ("", uri->get_fragment());
- EXPECT_EQ("", uri->get_query());
+ URI uri = expect_uri_nothrow("/foo");
+ EXPECT_EQ("", uri.get_scheme());
+ EXPECT_EQ("", uri.get_host());
+ EXPECT_FALSE(uri.has_port());
+ EXPECT_EQ(0, uri.get_port_or_default(0));
+ EXPECT_EQ("/foo", uri.get_path());
+ EXPECT_EQ("", uri.get_fragment());
+ EXPECT_EQ("", uri.get_query());
}
/* All fields */
{
- optional<URI> uri = URI::parse_from_string("hdfs://nn:8020/path/to/data?a=b&c=d#fragment");
- ASSERT_TRUE(uri && "Parsed");
- EXPECT_EQ("hdfs", uri->get_scheme());
- EXPECT_EQ("nn", uri->get_host());
- EXPECT_EQ(8020, uri->get_port().value_or(0));
- EXPECT_EQ("/path/to/data", uri->get_path());
- EXPECT_EQ("a=b&c=d", uri->get_query());
- EXPECT_EQ(3, uri->get_path_elements().size());
- EXPECT_EQ("path", uri->get_path_elements()[0]);
- EXPECT_EQ("to", uri->get_path_elements()[1]);
- EXPECT_EQ("data", uri->get_path_elements()[2]);
- EXPECT_EQ(2, uri->get_query_elements().size());
- EXPECT_EQ("a", uri->get_query_elements()[0].first);
- EXPECT_EQ("b", uri->get_query_elements()[0].second);
- EXPECT_EQ("c", uri->get_query_elements()[1].first);
- EXPECT_EQ("d", uri->get_query_elements()[1].second);
- EXPECT_EQ("fragment", uri->get_fragment());
+ URI uri = expect_uri_nothrow("hdfs://nn:8020/path/to/data?a=b&c=d#fragment");
+ EXPECT_EQ("hdfs", uri.get_scheme());
+ EXPECT_EQ("nn", uri.get_host());
+ EXPECT_TRUE(uri.has_port());
+ EXPECT_EQ(8020, uri.get_port());
+ EXPECT_EQ(8020, uri.get_port_or_default(0));
+ EXPECT_EQ("/path/to/data", uri.get_path());
+ EXPECT_EQ("a=b&c=d", uri.get_query());
+ EXPECT_EQ(3, uri.get_path_elements().size());
+ EXPECT_EQ("path", uri.get_path_elements()[0]);
+ EXPECT_EQ("to", uri.get_path_elements()[1]);
+ EXPECT_EQ("data", uri.get_path_elements()[2]);
+ EXPECT_EQ(2, uri.get_query_elements().size());
+ EXPECT_EQ("a", uri.get_query_elements()[0].key);
+ EXPECT_EQ("b", uri.get_query_elements()[0].value);
+ EXPECT_EQ("c", uri.get_query_elements()[1].key);
+ EXPECT_EQ("d", uri.get_query_elements()[1].value);
+ EXPECT_EQ("fragment", uri.get_fragment());
}
}
@@ -121,52 +145,48 @@ TEST(UriTest, TestEncodedInputs) {
/* Encoded input */
{
- optional<URI> uri = URI::parse_from_string("S://%5E:1/+%5E%20?%5E=%5E#%5E");
- ASSERT_TRUE(uri && "Parsed");
- EXPECT_EQ("S", uri->get_scheme());
- EXPECT_EQ("^", uri->get_host());
- EXPECT_EQ(1, uri->get_port().value_or(0));
- EXPECT_EQ("/ ^ ", uri->get_path());
- EXPECT_EQ("^", uri->get_fragment());
- EXPECT_EQ("^=^", uri->get_query());
+ URI uri = expect_uri_nothrow("S://%5E:1/+%5E%20?%5E=%5E#%5E");
+ EXPECT_EQ("S", uri.get_scheme());
+ EXPECT_EQ("^", uri.get_host());
+ EXPECT_EQ(1, uri.get_port_or_default(0));
+ EXPECT_EQ("/ ^ ", uri.get_path());
+ EXPECT_EQ("^", uri.get_fragment());
+ EXPECT_EQ("^=^", uri.get_query());
}
/* Lowercase */
{
- optional<URI> uri = URI::parse_from_string("S://%5e:1/+%5e%20?%5e=%5e#%5e");
- ASSERT_TRUE(uri && "Parsed");
- EXPECT_EQ("S", uri->get_scheme());
- EXPECT_EQ("^", uri->get_host());
- EXPECT_EQ(1, uri->get_port().value_or(0));
- EXPECT_EQ("/ ^ ", uri->get_path());
- EXPECT_EQ("^", uri->get_fragment());
- EXPECT_EQ("^=^", uri->get_query());
+ URI uri = expect_uri_nothrow("S://%5e:1/+%5e%20?%5e=%5e#%5e");
+ EXPECT_EQ("S", uri.get_scheme());
+ EXPECT_EQ("^", uri.get_host());
+ EXPECT_EQ(1, uri.get_port_or_default(0));
+ EXPECT_EQ("/ ^ ", uri.get_path());
+ EXPECT_EQ("^", uri.get_fragment());
+ EXPECT_EQ("^=^", uri.get_query());
}
}
TEST(UriTest, TestDecodedInputsAndOutputs) {
/* All fields non-encoded and shouldn't be interpreted */
{
- optional<URI> uri = URI::parse_from_string("S://%25/%25+?%25=%25#%25");
- ASSERT_TRUE(uri && "Parsed");
- EXPECT_EQ("S", uri->get_scheme());
- EXPECT_EQ("%", uri->get_host());
- EXPECT_EQ(0, uri->get_port().value_or(0));
- EXPECT_EQ("/% ", uri->get_path());
- EXPECT_EQ("%", uri->get_fragment());
- EXPECT_EQ("%=%", uri->get_query());
+ URI uri = expect_uri_nothrow("S://%25/%25+?%25=%25#%25");
+ EXPECT_EQ("S", uri.get_scheme());
+ EXPECT_EQ("%", uri.get_host());
+ EXPECT_EQ(0, uri.get_port_or_default(0));
+ EXPECT_EQ("/% ", uri.get_path());
+ EXPECT_EQ("%", uri.get_fragment());
+ EXPECT_EQ("%=%", uri.get_query());
}
/* All fields encode fields on their way out */
{
- optional<URI> uri = URI::parse_from_string("S://%25/%25+?%25=%25#%25");
- ASSERT_TRUE(uri && "Parsed");
- EXPECT_EQ("S", uri->get_scheme(true));
- EXPECT_EQ("%25", uri->get_host(true));
- EXPECT_EQ(0, uri->get_port().value_or(0));
- EXPECT_EQ("/%25+", uri->get_path(true));
- EXPECT_EQ("%25", uri->get_fragment(true));
- EXPECT_EQ("%25=%25", uri->get_query(true));
+ URI uri = expect_uri_nothrow("S://%25/%25+?%25=%25#%25");
+ EXPECT_EQ("S", uri.get_scheme(true));
+ EXPECT_EQ("%25", uri.get_host(true));
+ EXPECT_EQ(0, uri.get_port_or_default(0));
+ EXPECT_EQ("/%25+", uri.get_path(true));
+ EXPECT_EQ("%25", uri.get_fragment(true));
+ EXPECT_EQ("%25=%25", uri.get_query(true));
}
}
@@ -235,6 +255,35 @@ TEST(UriTest, TestSetters) {
}
+TEST(UriTest, QueryManip) {
+ // Not encoded, just basic adding and removing query parts
+ {
+ URI uri = URI::parse_from_string("hdfs://nn:8020/path?thedude=lebowski&donny=outofhiselement");
+ EXPECT_TRUE(uri.has_port());
+ EXPECT_EQ(uri.get_query(), "thedude=lebowski&donny=outofhiselement");
+
+ std::vector<URI::Query> queries = uri.get_query_elements();
+ EXPECT_EQ(queries.size(), 2);
+ EXPECT_EQ(queries[0].key, "thedude");
+ EXPECT_EQ(queries[0].value, "lebowski");
+ EXPECT_EQ(queries[1].key, "donny");
+ EXPECT_EQ(queries[1].value, "outofhiselement");
+
+ uri.remove_query("donny"); // that's a bummer, man
+ EXPECT_EQ(uri.get_query(), "thedude=lebowski");
+ queries = uri.get_query_elements();
+ EXPECT_EQ(queries.size(), 1);
+ EXPECT_EQ(queries[0].key, "thedude");
+ EXPECT_EQ(queries[0].value, "lebowski");
+
+ uri.add_query("HeyPeter", "CheckItOut");
+ EXPECT_EQ(uri.get_query(), "thedude=lebowski&HeyPeter=CheckItOut");
+ queries = uri.get_query_elements();
+ EXPECT_EQ(queries.size(), 2);
+ }
+
+}
+
int main(int argc, char *argv[]) {
/*
* The following line must be executed to initialize Google Mock
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_allowSnapshot.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_allowSnapshot.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_allowSnapshot.cc
index 00709e4..0293ee2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_allowSnapshot.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_allowSnapshot.cc
@@ -66,19 +66,15 @@ int main(int argc, char *argv[]) {
std::string uri_path = argv[optind];
//Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
+ hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
if (!fs) {
std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
}
- hdfs::Status status = fs->AllowSnapshot(uri->get_path());
+ hdfs::Status status = fs->AllowSnapshot(uri.get_path());
if (!status.ok()) {
std::cerr << "Error: " << status.ToString() << std::endl;
exit(EXIT_FAILURE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_cat.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_cat.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_cat.cc
index ec347cb..c89bb94 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_cat.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_cat.cc
@@ -67,19 +67,15 @@ int main(int argc, char *argv[]) {
std::string uri_path = argv[optind];
//Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
+ hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
if (!fs) {
std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
}
- readFile(fs, uri->get_path(), 0, stdout, false);
+ readFile(fs, uri.get_path(), 0, stdout, false);
// Clean up static data and prevent valgrind memory leaks
google::protobuf::ShutdownProtobufLibrary();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cc
index 3b3cd50..d64affe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cc
@@ -101,13 +101,9 @@ int main(int argc, char *argv[]) {
std::string uri_path = argv[optind + 1];
//Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
+ hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), true);
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, true);
if (!fs) {
std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
@@ -121,7 +117,7 @@ int main(int argc, char *argv[]) {
};
if(!recursive){
- fs->SetOwner(uri->get_path(), owner, group, handler);
+ fs->SetOwner(uri.get_path(), owner, group, handler);
}
else {
//Allocating shared state, which includes:
@@ -173,7 +169,7 @@ int main(int argc, char *argv[]) {
};
//Asynchronous call to Find
- fs->Find(uri->get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
+ fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
}
/* block until promise is set */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cc
index 69d61a5..091f18f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cc
@@ -97,13 +97,9 @@ int main(int argc, char *argv[]) {
std::string uri_path = argv[optind + 1];
//Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
+ hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), true);
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, true);
if (!fs) {
std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
@@ -119,7 +115,7 @@ int main(int argc, char *argv[]) {
//strtol() is reading the value with base 8, NULL because we are reading in just one value.
uint16_t perm = strtol(permissions.c_str(), NULL, 8);
if(!recursive){
- fs->SetPermission(uri->get_path(), perm, handler);
+ fs->SetPermission(uri.get_path(), perm, handler);
}
else {
//Allocating shared state, which includes:
@@ -171,7 +167,7 @@ int main(int argc, char *argv[]) {
};
//Asynchronous call to Find
- fs->Find(uri->get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
+ fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
}
/* block until promise is set */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cc
index 59ff9c3..24f7470 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cc
@@ -111,13 +111,9 @@ int main(int argc, char *argv[]) {
}
//Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
+ hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), true);
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, true);
if (!fs) {
std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
@@ -131,7 +127,7 @@ int main(int argc, char *argv[]) {
};
if(!recursive){
- fs->SetOwner(uri->get_path(), owner, group, handler);
+ fs->SetOwner(uri.get_path(), owner, group, handler);
}
else {
//Allocating shared state, which includes:
@@ -183,7 +179,7 @@ int main(int argc, char *argv[]) {
};
//Asynchronous call to Find
- fs->Find(uri->get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
+ fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
}
/* block until promise is set */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_copyToLocal.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_copyToLocal.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_copyToLocal.cc
index 493ff62..c9eb193 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_copyToLocal.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_copyToLocal.cc
@@ -66,13 +66,9 @@ int main(int argc, char *argv[]) {
std::string dest = argv[optind+1];
//Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
+ hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
if (!fs) {
std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
@@ -83,7 +79,7 @@ int main(int argc, char *argv[]) {
std::cerr << "Unable to open the destination file: " << dest << std::endl;
exit(EXIT_FAILURE);
}
- readFile(fs, uri->get_path(), 0, dst_file, false);
+ readFile(fs, uri.get_path(), 0, dst_file, false);
std::fclose(dst_file);
// Clean up static data and prevent valgrind memory leaks
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_count.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_count.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_count.cc
index e43596c..345ccc6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_count.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_count.cc
@@ -71,20 +71,16 @@ int main(int argc, char *argv[]) {
std::string uri_path = argv[optind];
//Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
+ hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
if (!fs) {
std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
}
hdfs::ContentSummary content_summary;
- hdfs::Status status = fs->GetContentSummary(uri->get_path(), content_summary);
+ hdfs::Status status = fs->GetContentSummary(uri.get_path(), content_summary);
if (!status.ok()) {
std::cerr << "Error: " << status.ToString() << std::endl;
exit(EXIT_FAILURE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_createSnapshot.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_createSnapshot.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_createSnapshot.cc
index fd079a2..1236dd6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_createSnapshot.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_createSnapshot.cc
@@ -75,19 +75,15 @@ int main(int argc, char *argv[]) {
std::string uri_path = argv[optind];
//Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
+ hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
if (!fs) {
std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
}
- hdfs::Status status = fs->CreateSnapshot(uri->get_path(), name);
+ hdfs::Status status = fs->CreateSnapshot(uri.get_path(), name);
if (!status.ok()) {
std::cerr << "Error: " << status.ToString() << std::endl;
exit(EXIT_FAILURE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_deleteSnapshot.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_deleteSnapshot.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_deleteSnapshot.cc
index 2082878..4cd76f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_deleteSnapshot.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_deleteSnapshot.cc
@@ -67,19 +67,15 @@ int main(int argc, char *argv[]) {
std::string name = argv[optind+1];
//Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
+ hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
if (!fs) {
std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
}
- hdfs::Status status = fs->DeleteSnapshot(uri->get_path(), name);
+ hdfs::Status status = fs->DeleteSnapshot(uri.get_path(), name);
if (!status.ok()) {
std::cerr << "Error: " << status.ToString() << std::endl;
exit(EXIT_FAILURE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_df.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_df.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_df.cc
index 6170e93..dddf010 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_df.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_df.cc
@@ -66,13 +66,9 @@ int main(int argc, char *argv[]) {
std::string uri_path = argv[optind];
//Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
+ hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
if (!fs) {
std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_disallowSnapshot.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_disallowSnapshot.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_disallowSnapshot.cc
index c181719..b5c97ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_disallowSnapshot.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_disallowSnapshot.cc
@@ -66,19 +66,15 @@ int main(int argc, char *argv[]) {
std::string uri_path = argv[optind];
//Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
+ hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
if (!fs) {
std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
}
- hdfs::Status status = fs->DisallowSnapshot(uri->get_path());
+ hdfs::Status status = fs->DisallowSnapshot(uri.get_path());
if (!status.ok()) {
std::cerr << "Error: " << status.ToString() << std::endl;
exit(EXIT_FAILURE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_du.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_du.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_du.cc
index 30269e3..f6b6e73 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_du.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_du.cc
@@ -92,13 +92,9 @@ int main(int argc, char *argv[]) {
std::string uri_path = argv[optind];
//Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
+ hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), true);
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, true);
if (!fs) {
std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
@@ -161,10 +157,10 @@ int main(int argc, char *argv[]) {
if(!recursive){
//Asynchronous call to Find
- fs->GetListing(uri->get_path(), handlerFind);
+ fs->GetListing(uri.get_path(), handlerFind);
} else {
//Asynchronous call to Find
- fs->Find(uri->get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
+ fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
}
/* block until promise is set */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_find.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_find.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_find.cc
index ef2ad7b..348f851 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_find.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_find.cc
@@ -92,13 +92,9 @@ int main(int argc, char *argv[]) {
std::string uri_path = argv[optind];
//Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
+ hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), true);
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, true);
if (!fs) {
std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
@@ -136,7 +132,7 @@ int main(int argc, char *argv[]) {
};
//Asynchronous call to Find
- fs->Find(uri->get_path(), name, max_depth, handler);
+ fs->Find(uri.get_path(), name, max_depth, handler);
//block until promise is set
future.get();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_get.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_get.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_get.cc
index f1ff1c8..16dd72d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_get.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_get.cc
@@ -66,13 +66,9 @@ int main(int argc, char *argv[]) {
std::string dest = argv[optind+1];
//Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
+ hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
if (!fs) {
std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
@@ -83,7 +79,7 @@ int main(int argc, char *argv[]) {
std::cerr << "Unable to open the destination file: " << dest << std::endl;
exit(EXIT_FAILURE);
}
- readFile(fs, uri->get_path(), 0, dst_file, false);
+ readFile(fs, uri.get_path(), 0, dst_file, false);
std::fclose(dst_file);
// Clean up static data and prevent valgrind memory leaks
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_ls.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_ls.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_ls.cc
index 710dbf7..e35e51d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_ls.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_ls.cc
@@ -71,13 +71,9 @@ int main(int argc, char *argv[]) {
std::string uri_path = argv[optind];
//Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
+ hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), true);
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, true);
if (!fs) {
std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
@@ -116,10 +112,10 @@ int main(int argc, char *argv[]) {
if(!recursive){
//Asynchronous call to GetListing
- fs->GetListing(uri->get_path(), handler);
+ fs->GetListing(uri.get_path(), handler);
} else {
//Asynchronous call to Find
- fs->Find(uri->get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handler);
+ fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handler);
}
//block until promise is set
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_mkdir.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_mkdir.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_mkdir.cc
index c4f08e3..3ccc601 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_mkdir.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_mkdir.cc
@@ -78,19 +78,15 @@ int main(int argc, char *argv[]) {
std::string uri_path = argv[optind];
//Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
+ hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
if (!fs) {
std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
}
- hdfs::Status status = fs->Mkdirs(uri->get_path(), permissions, create_parents);
+ hdfs::Status status = fs->Mkdirs(uri.get_path(), permissions, create_parents);
if (!status.ok()) {
std::cerr << "Error: " << status.ToString() << std::endl;
exit(EXIT_FAILURE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_moveToLocal.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_moveToLocal.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_moveToLocal.cc
index 5bb0754..0b83937 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_moveToLocal.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_moveToLocal.cc
@@ -68,13 +68,9 @@ int main(int argc, char *argv[]) {
std::string dest = argv[optind+1];
//Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
+ hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
if (!fs) {
std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
@@ -85,7 +81,7 @@ int main(int argc, char *argv[]) {
std::cerr << "Unable to open the destination file: " << dest << std::endl;
exit(EXIT_FAILURE);
}
- readFile(fs, uri->get_path(), 0, dst_file, true);
+ readFile(fs, uri.get_path(), 0, dst_file, true);
std::fclose(dst_file);
// Clean up static data and prevent valgrind memory leaks
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_renameSnapshot.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_renameSnapshot.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_renameSnapshot.cc
index 3262b34..8d38fc2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_renameSnapshot.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_renameSnapshot.cc
@@ -68,19 +68,15 @@ int main(int argc, char *argv[]) {
std::string new_name = argv[optind+2];
//Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
+ hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
if (!fs) {
std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
}
- hdfs::Status status = fs->RenameSnapshot(uri->get_path(), old_name, new_name);
+ hdfs::Status status = fs->RenameSnapshot(uri.get_path(), old_name, new_name);
if (!status.ok()) {
std::cerr << "Error: " << status.ToString() << std::endl;
exit(EXIT_FAILURE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_rm.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_rm.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_rm.cc
index ddea730..7056cf9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_rm.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_rm.cc
@@ -70,19 +70,15 @@ int main(int argc, char *argv[]) {
std::string uri_path = argv[optind];
//Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
+ hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), true);
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, true);
if (!fs) {
std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
}
- hdfs::Status status = fs->Delete(uri->get_path(), recursive);
+ hdfs::Status status = fs->Delete(uri.get_path(), recursive);
if (!status.ok()) {
std::cerr << "Error: " << status.ToString() << std::endl;
exit(EXIT_FAILURE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_setrep.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_setrep.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_setrep.cc
index 97aaee2..019e24d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_setrep.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_setrep.cc
@@ -90,13 +90,9 @@ int main(int argc, char *argv[]) {
std::string uri_path = argv[optind + 1];
//Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
+ hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), true);
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, true);
if (!fs) {
std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
@@ -161,7 +157,7 @@ int main(int argc, char *argv[]) {
};
//Asynchronous call to Find
- fs->Find(uri->get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
+ fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
/* block until promise is set */
hdfs::Status status = future.get();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_stat.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_stat.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_stat.cc
index c910571..59d513b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_stat.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_stat.cc
@@ -65,20 +65,16 @@ int main(int argc, char *argv[]) {
std::string uri_path = argv[optind];
//Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
+ hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
if (!fs) {
std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
}
hdfs::StatInfo stat_info;
- hdfs::Status status = fs->GetFileInfo(uri->get_path(), stat_info);
+ hdfs::Status status = fs->GetFileInfo(uri.get_path(), stat_info);
if (!status.ok()) {
std::cerr << "Error: " << status.ToString() << std::endl;
exit(EXIT_FAILURE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_tail.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_tail.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_tail.cc
index 2fc1969..8125bdc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_tail.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_tail.cc
@@ -73,13 +73,9 @@ int main(int argc, char *argv[]) {
std::string uri_path = argv[optind];
//Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
+ hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
if (!fs) {
std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
@@ -87,7 +83,7 @@ int main(int argc, char *argv[]) {
//We need to get the size of the file using stat
hdfs::StatInfo stat_info;
- hdfs::Status status = fs->GetFileInfo(uri->get_path(), stat_info);
+ hdfs::Status status = fs->GetFileInfo(uri.get_path(), stat_info);
if (!status.ok()) {
std::cerr << "Error: " << status.ToString() << std::endl;
exit(EXIT_FAILURE);
@@ -101,7 +97,7 @@ int main(int argc, char *argv[]) {
do {
off_t current_length = (off_t) stat_info.length;
- readFile(fs, uri->get_path(), offset, stdout, false);
+ readFile(fs, uri.get_path(), offset, stdout, false);
//Exit if -f flag was not set
if(!follow){
@@ -112,7 +108,7 @@ int main(int argc, char *argv[]) {
//Sleep for the REFRESH_RATE
sleep(REFRESH_RATE);
//Use stat to check the new filesize.
- status = fs->GetFileInfo(uri->get_path(), stat_info);
+ status = fs->GetFileInfo(uri.get_path(), stat_info);
if (!status.ok()) {
std::cerr << "Error: " << status.ToString() << std::endl;
exit(EXIT_FAILURE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc
index 133bbc9..5b0cf4e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc
@@ -47,7 +47,7 @@ namespace hdfs {
//Check if the user supplied the host
if(!uri.get_host().empty()){
//If port is supplied we use it, otherwise we use the empty string so that it will be looked up in configs.
- std::string port = (uri.get_port()) ? std::to_string(uri.get_port().value()) : "";
+ std::string port = uri.has_port() ? std::to_string(uri.get_port()) : "";
status = fs->Connect(uri.get_host(), port);
if (!status.ok()) {
std::cerr << "Could not connect to " << uri.get_host() << ":" << port << ". " << status.ToString() << std::endl;
@@ -112,4 +112,17 @@ namespace hdfs {
} while (last_bytes_read > 0);
return;
}
+
+
+ URI parse_path_or_exit(const std::string& path)
+ {
+ URI uri;
+ try {
+ uri = hdfs::URI::parse_from_string(path);
+ } catch (const uri_parse_error& e) {
+ std::cerr << "Malformed URI: " << path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ return uri;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e40bc7/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.h
index f7c5e09..7e725e4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.h
@@ -21,6 +21,7 @@
#define TOOLS_COMMON_H_
#include "hdfspp/hdfspp.h"
+#include "hdfspp/uri.h"
#include "common/hdfs_configuration.h"
#include "common/configuration_loader.h"
#include <mutex>
@@ -32,6 +33,8 @@ namespace hdfs {
//Open HDFS file at offset, read it to destination file, optionally delete source file
void readFile(std::shared_ptr<hdfs::FileSystem> fs, std::string path, off_t offset, std::FILE* dst_file, bool to_delete);
+
+ URI parse_path_or_exit(const std::string& path);
}
#endif
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/52] [abbrv] hadoop git commit: HDFS-10450b: libhdfs++: Add Cyrus
SASL support (engines). Contributed by Don Davis.
Posted by jh...@apache.org.
HDFS-10450b: libhdfs++: Add Cyrus SASL support (engines). Contributed by Don Davis.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d22e4b2e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d22e4b2e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d22e4b2e
Branch: refs/heads/HDFS-8707
Commit: d22e4b2eb77673725eece896bc52ccdce692d987
Parents: 549a5db
Author: Bob Hansen <bo...@hpe.com>
Authored: Fri Sep 9 14:58:16 2016 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../libhdfspp/lib/rpc/cyrus_sasl_engine.cc | 413 +++++++++++++++++++
.../libhdfspp/lib/rpc/cyrus_sasl_engine.h | 50 +++
.../native/libhdfspp/lib/rpc/gsasl_engine.cc | 182 ++++++++
.../native/libhdfspp/lib/rpc/gsasl_engine.h | 47 +++
4 files changed, 692 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22e4b2e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/cyrus_sasl_engine.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/cyrus_sasl_engine.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/cyrus_sasl_engine.cc
new file mode 100644
index 0000000..ca82305
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/cyrus_sasl_engine.cc
@@ -0,0 +1,413 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <sys/types.h>
+#include "sasl/sasl.h"
+#include "sasl/saslutil.h"
+#include <string.h>
+#include <string>
+#include <sstream>
+#include <unistd.h> // getpass() ( deprecated)
+
+#include "common/logging.h"
+
+#include "sasl_engine.h"
+#include "cyrus_sasl_engine.h"
+
+namespace hdfs {
+
+
+// Forward decls of sasl callback functions
+typedef int (*sasl_callback_ft)(void);
+int get_name(void *context,
+ int id,
+ const char **result,
+ unsigned *len);
+
+int getrealm(void *context,
+ int id,
+ const char **availrealms,
+ const char **result);
+
+// This should be constructed once per process, and destroyed once per process
+
+class CyrusPerProcessData
+{
+public:
+ static Status Init(); // Can be called many times
+private:
+ CyrusPerProcessData();
+ ~CyrusPerProcessData();
+ Status init_status_;
+
+ static CyrusPerProcessData & GetInstance();
+};
+
+
+/*****************************************************************************
+ * CYRUS UTILITY FUNCTIONS
+ */
+
+// Cyrus-specific error messages:
+// errStr() is the non-method version, to
+// be called by utility routines.
+std::string errStr( int rc) {
+ switch (rc) {
+ case SASL_NOTINIT: /* -12 */ return "SASL library not initialized";
+ case SASL_WRONGMECH:/* -11 */ return "mechanism doesn't support requested feature";
+ case SASL_BADSERV: /* -10 */ return "server failed mutual authentication step";
+ case SASL_BADMAC: /* -9 */ return "integrity check failed";
+ case SASL_TRYAGAIN: /* -8 */ return "transient failure (e.g., weak key)";
+ case SASL_BADPARAM: /* -7 */ return "invalid parameter supplied";
+ case SASL_NOTDONE: /* -6 */ return "can't request info until later in exchange";
+ case SASL_BADPROT: /* -5 */ return "bad protocol / cancel";
+ case SASL_NOMECH: /* -4 */ return "mechanism not supported";
+ case SASL_BUFOVER: /* -3 */ return "overflowed buffer";
+ case SASL_NOMEM: /* -2 */ return "memory shortage failure";
+ case SASL_FAIL: /* -1 */ return "generic failure";
+ case SASL_OK: /* 0 */ return "successful result";
+ case SASL_CONTINUE: /* 1 */ return "another step is needed in authentication";
+ case SASL_INTERACT: /* 2 */ return "needs user interaction";
+ default: return "unknown error";
+ } // switch(rc)
+} // errStr()
+
+Status make_status(int rc) {
+ if (rc != SASL_OK &&
+ rc != SASL_CONTINUE &&
+ rc != SASL_INTERACT) {
+ return Status::Error(errStr(rc).c_str());
+ }
+ return Status::OK();
+}
+
+// SaslError() method: Use this when a method needs
+// to update the engine's state.
+Status CySaslEngine::SaslError( int rc) {
+ Status status = make_status(rc);
+ if (!status.ok())
+ state_ = kErrorState;
+
+ return status;
+}
+
+
+/*****************************************************************************
+* Cyrus SASL ENGINE
+*/
+
+ CySaslEngine::CySaslEngine() : SaslEngine(), conn_(nullptr)
+ {
+ // Create an array of callbacks that embed a pointer to this
+ // so we can call methods of the engine
+ per_connection_callbacks_ = {
+ { SASL_CB_USER, (sasl_callback_ft) & get_name, this}, // userid for authZ
+ { SASL_CB_AUTHNAME, (sasl_callback_ft) & get_name, this}, // authid for authT
+ { SASL_CB_GETREALM, (sasl_callback_ft) & getrealm, this}, // krb/gssapi realm
+ // { SASL_CB_PASS, (sasl_callback_ft)&getsecret, this
+ { SASL_CB_LIST_END, (sasl_callback_ft) NULL, NULL}
+ };
+ }
+
+CySaslEngine::~CySaslEngine()
+{
+ if (conn_) {
+ sasl_dispose( &conn_); // undo sasl_client_new()
+ }
+} // destructor
+
+// initialize some cyrus sasl context stuff:
+
+Status CySaslEngine::InitCyrusSasl()
+{
+ int rc = SASL_OK;
+
+ // set up some callbacks once per process:
+ Status init_status = CyrusPerProcessData::Init();
+ if (!init_status.ok())
+ return init_status;
+
+ // Initialize the sasl_li brary with per-connection configuration:
+ const char * fqdn = chosen_mech_.serverid.c_str();
+ const char * proto = chosen_mech_.protocol.c_str();
+
+ rc = sasl_client_new(proto, fqdn, NULL, NULL, &per_connection_callbacks_[0], 0, &conn_);
+ if (rc != SASL_OK) return SaslError(rc);
+
+ return Status::OK();
+} // cysasl_new()
+
+// start() method: Ask the Sasl ibrary, "How do we
+// ask the hdfs server for service?
+std::pair<Status, std::string>
+CySaslEngine::Start()
+{
+ int rc;
+ Status status;
+
+ if (state_ != kUnstarted)
+ LOG_WARN(kRPC, << "CySaslEngine::start() when state is " << state_);
+
+ status = InitCyrusSasl();
+
+ if ( !status.ok()) {
+ state_ = kErrorState;
+ return std::make_pair( status, "");
+ }
+
+ sasl_interact_t * client_interact = NULL;
+ char * buf;
+ unsigned int buflen;
+ const char * chosen_mech;
+ std::string token;
+
+ rc = sasl_client_start(conn_, chosen_mech_.mechanism.c_str(), &client_interact,
+ (const char **) &buf, &buflen, &chosen_mech);
+
+ switch (rc) {
+ case SASL_OK: state_ = kSuccess;
+ break;
+ case SASL_CONTINUE: state_ = kWaitingForData;
+ break;
+ default: state_ = kFailure;
+ return std::make_pair( SaslError(rc), "");
+ break;
+ } // switch( rc)
+
+ // Cyrus will free this buffer when the connection is shut down
+ token = std::string( buf, buflen);
+ return std::make_pair( Status::OK(), token);
+} // start() method
+
+std::pair<Status, std::string> CySaslEngine::Step(const std::string data)
+{
+ char * output = NULL;
+ unsigned int outlen = 0;
+ sasl_interact_t * client_interact = NULL;
+
+ if (state_ != kWaitingForData)
+ LOG_WARN(kRPC, << "CySaslEngine::step when state is " << state_);
+
+ int rc = sasl_client_step(conn_, data.c_str(), data.size(), &client_interact,
+ (const char **) &output, &outlen);
+
+ // right now, state_ == kWaitingForData,
+ // so update state_, to reflect _step()'s result:
+ switch (rc) {
+ case SASL_OK: state_ = kSuccess; break;
+ case SASL_CONTINUE: state_ = kWaitingForData; break;
+ default: state_ = kFailure;
+ return std::make_pair(SaslError(rc), "");
+ break;
+ } // switch( rc)
+ return std::make_pair(Status::OK(), std::string( output,outlen));
+} // step() method
+
+Status CySaslEngine::Finish()
+{
+ if (state_ != kSuccess && state_ != kFailure && state_ != kErrorState )
+ LOG_WARN(kRPC, << "CySaslEngine::finish when state is " << state_);
+
+ if (conn_ != nullptr) {
+ sasl_dispose( &conn_);
+ conn_ = NULL;
+ }
+
+ return Status::OK();
+}
+
+//////////////////////////////////////////////////
+// Internal callbacks, for sasl_init_client(). //
+// Mostly lifted from cyrus' sample_client.c . //
+//////////////////////////////////////////////////
+
+static int
+sasl_my_log(void *context __attribute__((unused)),
+ int priority,
+ const char *message)
+{
+ if (! message)
+ return SASL_BADPARAM;
+
+ //TODO: get client, connection ID in here
+ switch (priority) {
+ case SASL_LOG_NONE: return SASL_OK; // no-op
+ case SASL_LOG_ERR: // fall through to FAIL
+ case SASL_LOG_FAIL:
+ LOG_ERROR(kRPC, << "SASL Error: " << message);
+ break;
+ case SASL_LOG_WARN:
+ LOG_ERROR(kRPC, << message);
+ break;
+ case SASL_LOG_NOTE:
+ LOG_INFO(kRPC, << message);
+ break;
+ case SASL_LOG_DEBUG:
+ LOG_DEBUG(kRPC, << message);
+ break;
+ case SASL_LOG_TRACE:
+ LOG_TRACE(kRPC, << message);
+ break;
+ case SASL_LOG_PASS: return SASL_OK; // don't log password-info
+ default:
+ LOG_WARN(kRPC, << "Unknown SASL log level(" << priority << "): " << message);
+ break;
+ }
+
+ return SASL_OK;
+} // sasl_my_log() callback
+
+static int
+sasl_getopt(void *context, const char *plugin_name,
+ const char *option,
+ const char **result, unsigned *len)
+{
+ if (plugin_name) {
+ LOG_WARN(kRPC, << "CySaslEngine: Unexpected plugin_name " << plugin_name);
+ return SASL_OK;
+ } // 123456789012345678
+ if (! strncmp( option, "canon_user_plugin", 18)) {
+ // TODO: maybe write a canon_user_plugin to do user-to-principal mapping
+ *result = "INTERNAL";
+ if (len) *len = strlen( *result);
+ return SASL_OK;
+ } // 12345678901234567
+ if (! strncmp( option, "client_mech_list", 17)) {
+ *result = "GSSAPI";
+ if (len) *len = strlen( *result);
+ return SASL_OK;
+ }
+
+ (void) context; // unused
+ return SASL_OK; }
+
+#define PLUGINDIR "/usr/local/lib/sasl2" // where the mechanisms are
+
+static int
+get_path(void *context, const char ** path)
+{
+ const char *searchpath = (const char *) context;
+
+ if (! path)
+ return SASL_BADPARAM;
+
+ // TODO: check the SASL_PATH environment, or will Cyrus pass that in in the context?
+ if (searchpath) {
+ *path = searchpath;
+ } else {
+ *path = PLUGINDIR;
+ }
+
+ return SASL_OK;
+} // getpath() callback
+
+int get_name(void *context,
+ int id,
+ const char **result,
+ unsigned *len)
+{
+ const CySaslEngine * pThis = (const CySaslEngine *) context;
+
+ if (!result)
+ return SASL_BADPARAM;
+
+ switch (id) {
+ case SASL_CB_AUTHNAME:
+ if (!pThis->id_)
+ break;
+ if (len)
+ *len = pThis->id_->size();
+ *result = pThis->id_->c_str();
+ break;
+ case SASL_CB_USER:
+ if (!pThis->principal_)
+ break;
+ if (len)
+ *len = pThis->principal_->size();
+ *result = pThis->principal_->c_str();
+ break;
+ case SASL_CB_LANGUAGE:
+ *result = NULL;
+ if (len)
+ *len = 0;
+ break;
+ default:
+ return SASL_BADPARAM;
+ }
+
+ LOG_DEBUG(kRPC, << "Cyrus::get_name: returning " << *result);
+
+ return SASL_OK;
+} // simple() callback
+
+int getrealm(void *context,
+ int id,
+ const char **availrealms,
+ const char **result)
+{
+ (void)availrealms; // unused
+ const CySaslEngine * pThis = (const CySaslEngine *) context;
+
+ if (!result)
+ return SASL_BADPARAM;
+
+ if (id != SASL_CB_GETREALM) return SASL_FAIL;
+ if (pThis->realm_)
+ *result = pThis->realm_->c_str();
+
+ return SASL_OK;
+} // getrealm() callback
+
+
+/*****************************************************************************
+* CYRUS PER-PROCESS INITIALIZATION
+*/
+
+
+const sasl_callback_t per_process_callbacks[] = {
+ { SASL_CB_LOG, (sasl_callback_ft) & sasl_my_log, NULL},
+ { SASL_CB_GETOPT, (sasl_callback_ft) & sasl_getopt, NULL},
+ { SASL_CB_GETPATH, (sasl_callback_ft) & get_path, NULL}, // to find th mechanisms
+ { SASL_CB_LIST_END, (sasl_callback_ft) NULL, NULL}
+ }; // callbacks_ array
+
+CyrusPerProcessData::CyrusPerProcessData()
+{
+ int init_rc = sasl_client_init(per_process_callbacks);
+ init_status_ = make_status(init_rc);
+}
+
+CyrusPerProcessData::~CyrusPerProcessData()
+{
+ // Undo sasl_client_init())
+ sasl_done();
+}
+
+Status CyrusPerProcessData::Init()
+{
+ return GetInstance().init_status_;
+}
+
+CyrusPerProcessData & CyrusPerProcessData::GetInstance()
+{
+ static CyrusPerProcessData per_process_data;
+ return per_process_data;
+}
+
+
+} // namespace hdfs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22e4b2e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/cyrus_sasl_engine.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/cyrus_sasl_engine.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/cyrus_sasl_engine.h
new file mode 100644
index 0000000..7c0f4e1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/cyrus_sasl_engine.h
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIB_RPC_CYRUS_SASLENGINE_H
+#define LIB_RPC_CYRUS_SASLENGINE_H
+
+#include "sasl/sasl.h"
+#include "sasl_engine.h"
+
+namespace hdfs
+{
+
+class CySaslEngine : public SaslEngine
+{
+public:
+ CySaslEngine();
+ virtual ~CySaslEngine();
+
+ virtual std::pair<Status, std::string> Start();
+ virtual std::pair<Status, std::string> Step(const std::string data);
+ virtual Status Finish();
+private:
+ Status InitCyrusSasl();
+ Status SaslError(int rc);
+
+ friend int get_name(void *, int, const char **, unsigned *);
+ friend int getrealm(void *, int, const char **availrealms, const char **);
+
+ sasl_conn_t * conn_;
+ std::vector<sasl_callback_t> per_connection_callbacks_;
+}; //class CySaslEngine
+
+} // namespace hdfs
+
+#endif /* LIB_RPC_CYRUS_SASLENGINE_H */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22e4b2e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/gsasl_engine.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/gsasl_engine.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/gsasl_engine.cc
new file mode 100644
index 0000000..8286bac
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/gsasl_engine.cc
@@ -0,0 +1,182 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <sstream>
+#include <gsasl.h>
+#include "sasl_engine.h"
+#include "gsasl_engine.h"
+#include "common/logging.h"
+
+namespace hdfs {
+
+/*****************************************************************************
+ * GSASL UTILITY FUNCTIONS
+ */
+
+static Status rc_to_status(int rc)
+{
+ if (rc == GSASL_OK) {
+ return Status::OK();
+ } else {
+ std::ostringstream ss;
+ ss << "Cannot initialize client (" << rc << "): " << gsasl_strerror(rc);
+ return Status::Error(ss.str().c_str());
+ }
+}
+
+static
+std::pair<Status, std::string> base64_encode(const std::string & in) {
+ char * temp;
+ size_t len;
+ std::string retval;
+ (void)base64_encode;
+
+ int rc = gsasl_base64_to(in.c_str(), in.size(), &temp, &len);
+
+ if (rc != GSASL_OK) {
+ return std::make_pair(rc_to_status(rc), "");
+ }
+
+ if (temp) {
+ retval = temp;
+ free(temp);
+ }
+
+ if (!temp || retval.length() != len) {
+ return std::make_pair(Status::Error("SaslEngine: Failed to encode string to base64"), "");
+ }
+
+ return std::make_pair(Status::OK(), retval);
+}
+
+/*****************************************************************************
+ * GSASL ENGINE
+ */
+
+GSaslEngine::~GSaslEngine()
+{
+ if (session_ != nullptr) {
+ gsasl_finish(session_);
+ }
+
+ if (ctx_ != nullptr) {
+ gsasl_done(ctx_);
+ }
+}
+
+Status GSaslEngine::gsasl_new() {
+ int status = GSASL_OK;
+
+ if (ctx_) return Status::OK();
+
+ status = gsasl_init( & ctx_);
+
+ switch ( status) {
+ case GSASL_OK:
+ return Status::OK();
+ case GSASL_MALLOC_ERROR:
+ LOG_WARN(kRPC, << "GSaslEngine: Out of memory.");
+ return Status::Error("SaslEngine: Out of memory.");
+ default:
+ LOG_WARN(kRPC, << "GSaslEngine: Unexpected error." << status);
+ return Status::Error("SaslEngine: Unexpected error.");
+ }
+} // gsasl_new()
+
+std::pair<Status, std::string>
+GSaslEngine::Start()
+{
+ int rc;
+ Status status;
+
+ this->gsasl_new();
+
+ /* Create new authentication session. */
+ rc = gsasl_client_start(ctx_, chosen_mech_.mechanism.c_str(), &session_);
+ if (rc != GSASL_OK) {
+ state_ = kErrorState;
+ return std::make_pair( rc_to_status( rc), std::string(""));
+ }
+ init_kerberos();
+
+ state_ = kWaitingForData;
+
+ // get from the sasl library the initial token
+ // that we'll send to the application server:
+ return this->Step( chosen_mech_.challenge.c_str());
+} // start() method
+
+Status GSaslEngine::init_kerberos() {
+
+ //TODO: check that we have a principal
+
+ gsasl_property_set(session_, GSASL_AUTHID, principal_.value().c_str());
+ gsasl_property_set(session_, GSASL_HOSTNAME, chosen_mech_.serverid.c_str());
+ gsasl_property_set(session_, GSASL_SERVICE, chosen_mech_.protocol.c_str());
+ return Status::OK();
+ }
+
+std::pair<Status, std::string> GSaslEngine::Step(const std::string data) {
+ if (state_ != kWaitingForData)
+ LOG_WARN(kRPC, << "GSaslEngine::step when state is " << state_);
+
+ char * output = NULL;
+ size_t outputSize;
+ int rc = gsasl_step(session_, data.c_str(), data.size(), &output,
+ &outputSize);
+
+ if (rc == GSASL_NEEDS_MORE || rc == GSASL_OK) {
+ std::string retval(output, output ? outputSize : 0);
+ if (output) {
+ free(output);
+ }
+
+ if (rc == GSASL_OK) {
+ state_ = kSuccess;
+ }
+
+ return std::make_pair(Status::OK(), retval);
+ }
+ else {
+ if (output) {
+ free(output);
+ }
+ state_ = kFailure;
+ return std::make_pair(rc_to_status(rc), "");
+ }
+}
+
+Status GSaslEngine::Finish()
+{
+ if (state_ != kSuccess && state_ != kFailure && state_ != kErrorState )
+ LOG_WARN(kRPC, << "GSaslEngine::finish when state is " << state_);
+
+ if (session_ != nullptr) {
+ gsasl_finish(session_);
+ session_ = NULL;
+ }
+
+ if (ctx_ != nullptr) {
+ gsasl_done(ctx_);
+ ctx_ = nullptr;
+ }
+
+ return Status::OK();
+} // finish() method
+
+} // namespace hdfs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22e4b2e/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/gsasl_engine.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/gsasl_engine.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/gsasl_engine.h
new file mode 100644
index 0000000..331b3fd
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/gsasl_engine.h
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIB_RPC_GSASLENGINE_H
+#define LIB_RPC_GSASLENGINE_H
+
+#include <gsasl.h>
+
+#include "sasl_engine.h"
+
+namespace hdfs {
+
+class GSaslEngine : public SaslEngine
+{
+public:
+ GSaslEngine() : SaslEngine(), ctx_(nullptr), session_(nullptr) {}
+ virtual ~GSaslEngine();
+
+ virtual std::pair<Status,std::string> Start();
+ virtual std::pair<Status,std::string> Step(const std::string data);
+ virtual Status Finish();
+private:
+ Status gsasl_new();
+ Gsasl * ctx_;
+ Gsasl_session * session_;
+
+ Status init_kerberos();
+};
+
+} // namespace hdfs
+
+#endif /* LIB_RPC_GSASLENGINE_H */
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/52] [abbrv] hadoop git commit: HDFS-9890: libhdfs++: Add test
suite to simulate network issues. Contributed by Xiaowei Zhu.
Posted by jh...@apache.org.
HDFS-9890: libhdfs++: Add test suite to simulate network issues. Contributed by Xiaowei Zhu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a586ccbc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a586ccbc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a586ccbc
Branch: refs/heads/HDFS-8707
Commit: a586ccbcfa8f3c3a851d311cf677d4565b0a1907
Parents: a23e6b4
Author: James <jh...@apache.org>
Authored: Wed Jul 13 15:36:14 2016 +0000
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../main/native/libhdfs-tests/native_mini_dfs.c | 22 +-
.../main/native/libhdfs-tests/native_mini_dfs.h | 11 +-
.../libhdfs-tests/test_libhdfs_mini_stress.c | 345 +++++++++++++++++++
.../native/libhdfspp/include/hdfspp/events.h | 6 +-
.../native/libhdfspp/include/hdfspp/hdfs_ext.h | 4 +-
.../native/libhdfspp/lib/bindings/c/hdfs.cc | 7 +-
.../main/native/libhdfspp/lib/fs/filehandle.cc | 15 +-
.../main/native/libhdfspp/lib/fs/filehandle.h | 3 +-
.../native/libhdfspp/lib/reader/block_reader.cc | 62 +++-
.../native/libhdfspp/lib/reader/block_reader.h | 5 +-
.../native/libhdfspp/lib/rpc/rpc_connection.cc | 13 +-
.../native/libhdfspp/lib/rpc/rpc_connection.h | 29 +-
.../main/native/libhdfspp/lib/rpc/rpc_engine.h | 1 -
.../main/native/libhdfspp/tests/CMakeLists.txt | 7 +
.../native/libhdfspp/tests/bad_datanode_test.cc | 5 +-
.../native/libhdfspp/tests/rpc_engine_test.cc | 2 +-
16 files changed, 483 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a586ccbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/native_mini_dfs.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/native_mini_dfs.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/native_mini_dfs.c
index b36ef76..6938109 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/native_mini_dfs.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/native_mini_dfs.c
@@ -182,6 +182,16 @@ struct NativeMiniDfsCluster* nmdCreate(struct NativeMiniDfsConf *conf)
}
(*env)->DeleteLocalRef(env, val.l);
}
+ if (conf->numDataNodes) {
+ jthr = invokeMethod(env, &val, INSTANCE, bld, MINIDFS_CLUSTER_BUILDER,
+ "numDataNodes", "(I)L" MINIDFS_CLUSTER_BUILDER ";", conf->numDataNodes);
+ if (jthr) {
+ printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "nmdCreate: "
+ "Builder::numDataNodes");
+ goto error;
+ }
+ }
+ (*env)->DeleteLocalRef(env, val.l);
jthr = invokeMethod(env, &val, INSTANCE, bld, MINIDFS_CLUSTER_BUILDER,
"build", "()L" MINIDFS_CLUSTER ";");
if (jthr) {
@@ -291,7 +301,7 @@ int nmdGetNameNodeHttpAddress(const struct NativeMiniDfsCluster *cl,
jthrowable jthr;
int ret = 0;
const char *host;
-
+
if (!env) {
fprintf(stderr, "nmdHdfsConnect: getJNIEnv failed\n");
return -EIO;
@@ -306,7 +316,7 @@ int nmdGetNameNodeHttpAddress(const struct NativeMiniDfsCluster *cl,
return -EIO;
}
jNameNode = jVal.l;
-
+
// Then get the http address (InetSocketAddress) of the NameNode
jthr = invokeMethod(env, &jVal, INSTANCE, jNameNode, HADOOP_NAMENODE,
"getHttpAddress", "()L" JAVA_INETSOCKETADDRESS ";");
@@ -317,7 +327,7 @@ int nmdGetNameNodeHttpAddress(const struct NativeMiniDfsCluster *cl,
goto error_dlr_nn;
}
jAddress = jVal.l;
-
+
jthr = invokeMethod(env, &jVal, INSTANCE, jAddress,
JAVA_INETSOCKETADDRESS, "getPort", "()I");
if (jthr) {
@@ -327,7 +337,7 @@ int nmdGetNameNodeHttpAddress(const struct NativeMiniDfsCluster *cl,
goto error_dlr_addr;
}
*port = jVal.i;
-
+
jthr = invokeMethod(env, &jVal, INSTANCE, jAddress, JAVA_INETSOCKETADDRESS,
"getHostName", "()Ljava/lang/String;");
if (jthr) {
@@ -339,12 +349,12 @@ int nmdGetNameNodeHttpAddress(const struct NativeMiniDfsCluster *cl,
host = (*env)->GetStringUTFChars(env, jVal.l, NULL);
*hostName = strdup(host);
(*env)->ReleaseStringUTFChars(env, jVal.l, host);
-
+
error_dlr_addr:
(*env)->DeleteLocalRef(env, jAddress);
error_dlr_nn:
(*env)->DeleteLocalRef(env, jNameNode);
-
+
return ret;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a586ccbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/native_mini_dfs.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/native_mini_dfs.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/native_mini_dfs.h
index ce8b1cf..628180f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/native_mini_dfs.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/native_mini_dfs.h
@@ -26,7 +26,7 @@ extern "C" {
#endif
struct hdfsBuilder;
-struct NativeMiniDfsCluster;
+struct NativeMiniDfsCluster;
/**
* Represents a configuration to use for creating a Native MiniDFSCluster
@@ -51,6 +51,11 @@ struct NativeMiniDfsConf {
* Nonzero if we should configure short circuit.
*/
jboolean configureShortCircuit;
+
+ /**
+ * The number of datanodes in MiniDfsCluster
+ */
+ jint numDataNodes;
};
/**
@@ -96,13 +101,13 @@ void nmdFree(struct NativeMiniDfsCluster* cl);
*
* @return the port, or a negative error code
*/
-int nmdGetNameNodePort(const struct NativeMiniDfsCluster *cl);
+int nmdGetNameNodePort(const struct NativeMiniDfsCluster *cl);
/**
* Get the http address that's in use by the given (non-HA) nativeMiniDfs
*
* @param cl The initialized NativeMiniDfsCluster
- * @param port Used to capture the http port of the NameNode
+ * @param port Used to capture the http port of the NameNode
* of the NativeMiniDfsCluster
* @param hostName Used to capture the http hostname of the NameNode
* of the NativeMiniDfsCluster
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a586ccbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c
new file mode 100644
index 0000000..0d01e44
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c
@@ -0,0 +1,345 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "expect.h"
+#include "hdfs/hdfs.h"
+#include "hdfspp/hdfs_ext.h"
+#include "native_mini_dfs.h"
+#include "os/thread.h"
+
+#include <errno.h>
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define TO_STR_HELPER(X) #X
+#define TO_STR(X) TO_STR_HELPER(X)
+
+#define TLH_MAX_THREADS 10000
+
+#define TLH_MAX_DNS 16
+
+#define TLH_DEFAULT_BLOCK_SIZE 1048576
+
+#define TLH_DEFAULT_DFS_REPLICATION 3
+
+#define TLH_DEFAULT_IPC_CLIENT_CONNECT_MAX_RETRIES 100
+
+#define TLH_DEFAULT_IPC_CLIENT_CONNECT_RETRY_INTERVAL_MS 5
+
+#ifndef RANDOM_ERROR_RATIO
+#define RANDOM_ERROR_RATIO 1000000000
+#endif
+
+struct tlhThreadInfo {
+ /** Thread index */
+ int threadIdx;
+ /** 0 = thread was successful; error code otherwise */
+ int success;
+ /** thread identifier */
+ thread theThread;
+ /** fs, shared with other threads **/
+ hdfsFS hdfs;
+ /** Filename */
+ const char *fileNm;
+
+};
+
+static int hdfsNameNodeConnect(struct NativeMiniDfsCluster *cl, hdfsFS *fs,
+ const char *username)
+{
+ int ret;
+ tPort port;
+ hdfsFS hdfs;
+ struct hdfsBuilder *bld;
+
+ port = (tPort)nmdGetNameNodePort(cl);
+ if (port < 0) {
+ fprintf(stderr, "hdfsNameNodeConnect: nmdGetNameNodePort "
+ "returned error %d\n", port);
+ return port;
+ }
+ bld = hdfsNewBuilder();
+ if (!bld)
+ return -ENOMEM;
+ hdfsBuilderSetForceNewInstance(bld);
+ hdfsBuilderSetNameNode(bld, "localhost");
+ hdfsBuilderSetNameNodePort(bld, port);
+ hdfsBuilderConfSetStr(bld, "dfs.block.size",
+ TO_STR(TLH_DEFAULT_BLOCK_SIZE));
+ hdfsBuilderConfSetStr(bld, "dfs.blocksize",
+ TO_STR(TLH_DEFAULT_BLOCK_SIZE));
+ hdfsBuilderConfSetStr(bld, "dfs.replication",
+ TO_STR(TLH_DEFAULT_DFS_REPLICATION));
+ hdfsBuilderConfSetStr(bld, "ipc.client.connect.max.retries",
+ TO_STR(TLH_DEFAULT_IPC_CLIENT_CONNECT_MAX_RETRIES));
+ hdfsBuilderConfSetStr(bld, "ipc.client.connect.retry.interval",
+ TO_STR(TLH_DEFAULT_IPC_CLIENT_CONNECT_RETRY_INTERVAL_MS));
+ if (username) {
+ hdfsBuilderSetUserName(bld, username);
+ }
+ hdfs = hdfsBuilderConnect(bld);
+ if (!hdfs) {
+ ret = -errno;
+ return ret;
+ }
+ *fs = hdfs;
+ return 0;
+}
+
+static int hdfsWriteData(hdfsFS hdfs, const char *dirNm,
+ const char *fileNm, tSize fileSz)
+{
+ hdfsFile file;
+ int ret, expected;
+ const char *content;
+
+ content = fileNm;
+
+ if (hdfsExists(hdfs, dirNm) == 0) {
+ EXPECT_ZERO(hdfsDelete(hdfs, dirNm, 1));
+ }
+ EXPECT_ZERO(hdfsCreateDirectory(hdfs, dirNm));
+
+ file = hdfsOpenFile(hdfs, fileNm, O_WRONLY, 0, 0, 0);
+ EXPECT_NONNULL(file);
+
+ expected = (int)strlen(content);
+ tSize sz = 0;
+ while (sz < fileSz) {
+ ret = hdfsWrite(hdfs, file, content, expected);
+ if (ret < 0) {
+ ret = errno;
+ fprintf(stderr, "hdfsWrite failed and set errno %d\n", ret);
+ return ret;
+ }
+ if (ret != expected) {
+ fprintf(stderr, "hdfsWrite was supposed to write %d bytes, but "
+ "it wrote %d\n", ret, expected);
+ return EIO;
+ }
+ sz += ret;
+ }
+ EXPECT_ZERO(hdfsFlush(hdfs, file));
+ EXPECT_ZERO(hdfsHSync(hdfs, file));
+ EXPECT_ZERO(hdfsCloseFile(hdfs, file));
+ return 0;
+}
+
+static int fileEventCallback1(const char * event, const char * cluster, const char * file, int64_t value, int64_t cookie)
+{
+ char * randomErrRatioStr = getenv("RANDOM_ERROR_RATIO");
+ int64_t randomErrRatio = RANDOM_ERROR_RATIO;
+ if (randomErrRatioStr) randomErrRatio = (int64_t)atoi(randomErrRatioStr);
+ if (randomErrRatio == 0) return DEBUG_SIMULATE_ERROR;
+ else if (randomErrRatio < 0) return LIBHDFSPP_EVENT_OK;
+ return random() % randomErrRatio == 0 ? DEBUG_SIMULATE_ERROR : LIBHDFSPP_EVENT_OK;
+}
+
+static int fileEventCallback2(const char * event, const char * cluster, const char * file, int64_t value, int64_t cookie)
+{
+ /* no op */
+ return LIBHDFSPP_EVENT_OK;
+}
+
+static int doTestHdfsMiniStress(struct tlhThreadInfo *ti, int randomErr)
+{
+ char tmp[4096];
+ hdfsFile file;
+ int ret, expected;
+ hdfsFileInfo *fileInfo;
+ uint64_t readOps, nErrs=0;
+ tOffset seekPos;
+ const char *content;
+
+ content = ti->fileNm;
+ expected = (int)strlen(content);
+
+ fileInfo = hdfsGetPathInfo(ti->hdfs, ti->fileNm);
+ EXPECT_NONNULL(fileInfo);
+
+ file = hdfsOpenFile(ti->hdfs, ti->fileNm, O_RDONLY, 0, 0, 0);
+ EXPECT_NONNULL(file);
+
+ libhdfspp_file_event_callback callback = (randomErr != 0) ? &fileEventCallback1 : &fileEventCallback2;
+
+ hdfsPreAttachFileMonitor(callback, 0);
+
+ fprintf(stderr, "testHdfsMiniStress(threadIdx=%d): starting read loop\n",
+ ti->threadIdx);
+ for (readOps=0; readOps < 1000; ++readOps) {
+ EXPECT_ZERO(hdfsCloseFile(ti->hdfs, file));
+ file = hdfsOpenFile(ti->hdfs, ti->fileNm, O_RDONLY, 0, 0, 0);
+ EXPECT_NONNULL(file);
+ seekPos = (((double)random()) / RAND_MAX) * (fileInfo->mSize - expected);
+ seekPos = (seekPos / expected) * expected;
+ ret = hdfsSeek(ti->hdfs, file, seekPos);
+ if (ret < 0) {
+ ret = errno;
+ fprintf(stderr, "hdfsSeek to %"PRIu64" failed and set"
+ " errno %d\n", seekPos, ret);
+ ++nErrs;
+ continue;
+ }
+ ret = hdfsRead(ti->hdfs, file, tmp, expected);
+ if (ret < 0) {
+ ret = errno;
+ fprintf(stderr, "hdfsRead failed and set errno %d\n", ret);
+ ++nErrs;
+ continue;
+ }
+ if (ret != expected) {
+ fprintf(stderr, "hdfsRead was supposed to read %d bytes, but "
+ "it read %d\n", ret, expected);
+ ++nErrs;
+ continue;
+ }
+ ret = memcmp(content, tmp, expected);
+ if (ret) {
+ fprintf(stderr, "hdfsRead result (%.*s) does not match expected (%.*s)",
+ expected, tmp, expected, content);
+ ++nErrs;
+ continue;
+ }
+ }
+ EXPECT_ZERO(hdfsCloseFile(ti->hdfs, file));
+ fprintf(stderr, "testHdfsMiniStress(threadIdx=%d): finished read loop\n",
+ ti->threadIdx);
+ EXPECT_ZERO(nErrs);
+ return 0;
+}
+
+static int testHdfsMiniStressImpl(struct tlhThreadInfo *ti)
+{
+ fprintf(stderr, "testHdfsMiniStress(threadIdx=%d): starting\n",
+ ti->threadIdx);
+ EXPECT_NONNULL(ti->hdfs);
+ EXPECT_ZERO(doTestHdfsMiniStress(ti, 1));
+ EXPECT_ZERO(doTestHdfsMiniStress(ti, 0));
+ return 0;
+}
+
+static void testHdfsMiniStress(void *v)
+{
+ struct tlhThreadInfo *ti = (struct tlhThreadInfo*)v;
+ int ret = testHdfsMiniStressImpl(ti);
+ ti->success = ret;
+}
+
+static int checkFailures(struct tlhThreadInfo *ti, int tlhNumThreads)
+{
+ int i, threadsFailed = 0;
+ const char *sep = "";
+
+ for (i = 0; i < tlhNumThreads; i++) {
+ if (ti[i].success != 0) {
+ threadsFailed = 1;
+ }
+ }
+ if (!threadsFailed) {
+ fprintf(stderr, "testLibHdfsMiniStress: all threads succeeded. SUCCESS.\n");
+ return EXIT_SUCCESS;
+ }
+ fprintf(stderr, "testLibHdfsMiniStress: some threads failed: [");
+ for (i = 0; i < tlhNumThreads; i++) {
+ if (ti[i].success != 0) {
+ fprintf(stderr, "%s%d", sep, i);
+ sep = ", ";
+ }
+ }
+ fprintf(stderr, "]. FAILURE.\n");
+ return EXIT_FAILURE;
+}
+
+/**
+ * Test intended to stress libhdfs client with concurrent requests. Currently focused
+ * on concurrent reads.
+ */
+int main(void)
+{
+ int i, tlhNumThreads;
+ char *dirNm, *fileNm;
+ tSize fileSz;
+ const char *tlhNumThreadsStr, *tlhNumDNsStr;
+ hdfsFS hdfs = NULL;
+ struct NativeMiniDfsCluster* tlhCluster;
+ struct tlhThreadInfo ti[TLH_MAX_THREADS];
+ struct NativeMiniDfsConf conf = {
+ 1, /* doFormat */
+ };
+
+ dirNm = "/tlhMiniStressData";
+ fileNm = "/tlhMiniStressData/file";
+ fileSz = 2*1024*1024;
+
+ tlhNumDNsStr = getenv("TLH_NUM_DNS");
+ if (!tlhNumDNsStr) {
+ tlhNumDNsStr = "1";
+ }
+ conf.numDataNodes = atoi(tlhNumDNsStr);
+ if ((conf.numDataNodes <= 0) || (conf.numDataNodes > TLH_MAX_DNS)) {
+ fprintf(stderr, "testLibHdfsMiniStress: must have a number of datanodes "
+ "between 1 and %d inclusive, not %d\n",
+ TLH_MAX_DNS, conf.numDataNodes);
+ return EXIT_FAILURE;
+ }
+
+ tlhNumThreadsStr = getenv("TLH_NUM_THREADS");
+ if (!tlhNumThreadsStr) {
+ tlhNumThreadsStr = "8";
+ }
+ tlhNumThreads = atoi(tlhNumThreadsStr);
+ if ((tlhNumThreads <= 0) || (tlhNumThreads > TLH_MAX_THREADS)) {
+ fprintf(stderr, "testLibHdfsMiniStress: must have a number of threads "
+ "between 1 and %d inclusive, not %d\n",
+ TLH_MAX_THREADS, tlhNumThreads);
+ return EXIT_FAILURE;
+ }
+ memset(&ti[0], 0, sizeof(ti));
+ for (i = 0; i < tlhNumThreads; i++) {
+ ti[i].threadIdx = i;
+ }
+
+ tlhCluster = nmdCreate(&conf);
+ EXPECT_NONNULL(tlhCluster);
+ EXPECT_ZERO(nmdWaitClusterUp(tlhCluster));
+
+ EXPECT_ZERO(hdfsNameNodeConnect(tlhCluster, &hdfs, NULL));
+
+ // Single threaded writes for now.
+ EXPECT_ZERO(hdfsWriteData(hdfs, dirNm, fileNm, fileSz));
+
+ // Multi-threaded reads.
+ for (i = 0; i < tlhNumThreads; i++) {
+ ti[i].theThread.start = testHdfsMiniStress;
+ ti[i].theThread.arg = &ti[i];
+ ti[i].hdfs = hdfs;
+ ti[i].fileNm = fileNm;
+ EXPECT_ZERO(threadCreate(&ti[i].theThread));
+ }
+ for (i = 0; i < tlhNumThreads; i++) {
+ EXPECT_ZERO(threadJoin(&ti[i].theThread));
+ }
+
+ EXPECT_ZERO(hdfsDisconnect(hdfs));
+ EXPECT_ZERO(nmdShutdown(tlhCluster));
+ nmdFree(tlhCluster);
+ return checkFailures(ti, tlhNumThreads);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a586ccbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/events.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/events.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/events.h
index 82109fd..43187a5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/events.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/events.h
@@ -48,10 +48,8 @@ public:
enum event_response_type {
kOk = 0,
-#ifndef NDEBUG
// Responses to be used in testing only
kTest_Error = 100
-#endif
};
@@ -70,10 +68,9 @@ private:
//
// Testing support
//
-// If running a debug build, the consumer can stimulate errors
+// The consumer can stimulate errors
// within libhdfdspp by returning a Status from the callback.
///////////////////////////////////////////////
-#ifndef NDEBUG
public:
static event_response test_err(const Status &status) {
return event_response(status);
@@ -86,7 +83,6 @@ private:
response_(event_response_type::kTest_Error), error_status_(status) {}
Status error_status_; // To be used with kTest_Error
-#endif
};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a586ccbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
index af7393f..6ec3a4b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
@@ -235,9 +235,7 @@ extern const char * FILE_DN_WRITE_EVENT;
#define LIBHDFSPP_EVENT_OK (0)
-#ifndef NDEBUG
- #define DEBUG_SIMULATE_ERROR (-1)
-#endif
+#define DEBUG_SIMULATE_ERROR (-1)
typedef int (*libhdfspp_fs_event_callback)(const char * event, const char * cluster,
int64_t value, int64_t cookie);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a586ccbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
index 04065b2..a42feae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
@@ -331,6 +331,9 @@ hdfsFile hdfsOpenFile(hdfsFS fs, const char *path, int flags, int bufferSize,
Error(stat);
return nullptr;
}
+ if (f && fileEventCallback) {
+ f->SetFileEventCallback(fileEventCallback.value());
+ }
return new hdfsFile_internal(f);
} catch (const std::exception & e) {
ReportException(e);
@@ -959,7 +962,7 @@ event_response fs_callback_glue(libhdfspp_fs_event_callback handler,
if (result == LIBHDFSPP_EVENT_OK) {
return event_response::ok();
}
-#ifndef NDEBUG
+#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
if (result == DEBUG_SIMULATE_ERROR) {
return event_response::test_err(Status::Error("Simulated error"));
}
@@ -978,7 +981,7 @@ event_response file_callback_glue(libhdfspp_file_event_callback handler,
if (result == LIBHDFSPP_EVENT_OK) {
return event_response::ok();
}
-#ifndef NDEBUG
+#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
if (result == DEBUG_SIMULATE_ERROR) {
return event_response::test_err(Status::Error("Simulated error"));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a586ccbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
index 38d50f6..df147d3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
@@ -239,7 +239,7 @@ void FileHandleImpl::AsyncPreadSome(
// Wrap the DN in a block reader to handle the state and logic of the
// block request protocol
std::shared_ptr<BlockReader> reader;
- reader = CreateBlockReader(BlockReaderOptions(), dn);
+ reader = CreateBlockReader(BlockReaderOptions(), dn, event_handlers_);
// Lambdas cannot capture copies of member variables so we'll make explicit
// copies for it
@@ -248,8 +248,8 @@ void FileHandleImpl::AsyncPreadSome(
auto cluster_name = cluster_name_;
auto read_handler = [reader, event_handlers, cluster_name, path, dn_id, handler](const Status & status, size_t transferred) {
- auto event_resp = event_handlers->call(FILE_DN_READ_EVENT, cluster_name.c_str(), path.c_str(), transferred);
-#ifndef NDEBUG
+ event_response event_resp = event_handlers->call(FILE_DN_READ_EVENT, cluster_name.c_str(), path.c_str(), transferred);
+#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
if (event_resp.response() == event_response::kTest_Error) {
handler(event_resp.status(), dn_id, transferred);
return;
@@ -262,8 +262,8 @@ void FileHandleImpl::AsyncPreadSome(
auto connect_handler = [handler,event_handlers,cluster_name,path,read_handler,block,offset_within_block,size_within_block, buffers, reader, dn_id, client_name]
(Status status, std::shared_ptr<DataNodeConnection> dn) {
(void)dn;
- auto event_resp = event_handlers->call(FILE_DN_CONNECT_EVENT, cluster_name.c_str(), path.c_str(), 0);
-#ifndef NDEBUG
+ event_response event_resp = event_handlers->call(FILE_DN_CONNECT_EVENT, cluster_name.c_str(), path.c_str(), 0);
+#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
if (event_resp.response() == event_response::kTest_Error) {
status = event_resp.status();
}
@@ -284,9 +284,10 @@ void FileHandleImpl::AsyncPreadSome(
}
std::shared_ptr<BlockReader> FileHandleImpl::CreateBlockReader(const BlockReaderOptions &options,
- std::shared_ptr<DataNodeConnection> dn)
+ std::shared_ptr<DataNodeConnection> dn,
+ std::shared_ptr<LibhdfsEvents> event_handlers)
{
- std::shared_ptr<BlockReader> reader = std::make_shared<BlockReaderImpl>(options, dn, cancel_state_);
+ std::shared_ptr<BlockReader> reader = std::make_shared<BlockReaderImpl>(options, dn, cancel_state_, event_handlers);
LOG_TRACE(kFileHandle, << "FileHandleImpl::CreateBlockReader(" << FMT_THIS_ADDR
<< ", ..., dnconn=" << dn.get()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a586ccbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
index a99550a..57cf4b7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
@@ -119,7 +119,8 @@ public:
protected:
virtual std::shared_ptr<BlockReader> CreateBlockReader(const BlockReaderOptions &options,
- std::shared_ptr<DataNodeConnection> dn);
+ std::shared_ptr<DataNodeConnection> dn,
+ std::shared_ptr<hdfs::LibhdfsEvents> event_handlers);
virtual std::shared_ptr<DataNodeConnection> CreateDataNodeConnection(
::asio::io_service *io_service,
const ::hadoop::hdfs::DatanodeInfoProto & dn,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a586ccbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc
index 5052951..6098b9c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc
@@ -24,7 +24,6 @@
#include <future>
-
namespace hdfs {
#define FMT_CONT_AND_PARENT_ADDR "this=" << (void*)this << ", parent=" << (void*)parent_
@@ -105,7 +104,17 @@ void BlockReaderImpl::AsyncRequestBlock(
m->Run([this, handler, offset](const Status &status, const State &s) { Status stat = status;
if (stat.ok()) {
const auto &resp = s.response;
- if (resp.status() == ::hadoop::hdfs::Status::SUCCESS) {
+
+ if(this->event_handlers_) {
+ event_response event_resp = this->event_handlers_->call(FILE_DN_READ_EVENT, "", "", 0);
+#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
+ if (stat.ok() && event_resp.response() == event_response::kTest_Error) {
+ stat = Status::Error("Test error");
+ }
+#endif
+ }
+
+ if (stat.ok() && resp.status() == ::hadoop::hdfs::Status::SUCCESS) {
if (resp.has_readopchecksuminfo()) {
const auto &checksum_info = resp.readopchecksuminfo();
chunk_padding_bytes_ = offset - checksum_info.chunkoffset();
@@ -162,6 +171,14 @@ struct BlockReaderImpl::ReadPacketHeader
assert(v && "Failed to parse the header");
parent_->state_ = kReadChecksum;
}
+ if(parent_->event_handlers_) {
+ event_response event_resp = parent_->event_handlers_->call(FILE_DN_READ_EVENT, "", "", 0);
+#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
+ if (status.ok() && event_resp.response() == event_response::kTest_Error) {
+ status = Status::Error("Test error");
+ }
+#endif
+ }
next(status);
};
@@ -214,7 +231,7 @@ struct BlockReaderImpl::ReadChecksum : continuation::Continuation {
return;
}
- auto handler = [parent, next](const asio::error_code &ec, size_t) {
+ auto handler = [parent, next, this](const asio::error_code &ec, size_t) {
Status status;
if (ec) {
status = Status(ec.value(), ec.message().c_str());
@@ -222,6 +239,14 @@ struct BlockReaderImpl::ReadChecksum : continuation::Continuation {
parent->state_ =
parent->chunk_padding_bytes_ ? kReadPadding : kReadData;
}
+ if(parent->event_handlers_) {
+ event_response event_resp = parent->event_handlers_->call(FILE_DN_READ_EVENT, "", "", 0);
+#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
+ if (status.ok() && event_resp.response() == event_response::kTest_Error) {
+ status = Status::Error("Test error");
+ }
+#endif
+ }
next(status);
};
parent->checksum_.resize(parent->packet_len_ - sizeof(int) -
@@ -248,7 +273,6 @@ struct BlockReaderImpl::ReadData : continuation::Continuation {
virtual void Run(const Next &next) override {
LOG_TRACE(kBlockReader, << "BlockReaderImpl::ReadData::Run("
<< FMT_CONT_AND_PARENT_ADDR << ") called");
-
auto handler =
[next, this](const asio::error_code &ec, size_t transferred) {
Status status;
@@ -261,6 +285,14 @@ struct BlockReaderImpl::ReadData : continuation::Continuation {
if (parent_->packet_data_read_bytes_ >= parent_->header_.datalen()) {
parent_->state_ = kReadPacketHeader;
}
+ if(parent_->event_handlers_) {
+ event_response event_resp = parent_->event_handlers_->call(FILE_DN_READ_EVENT, "", "", 0);
+#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
+ if (status.ok() && event_resp.response() == event_response::kTest_Error) {
+ status = Status::Error("Test error");
+ }
+#endif
+ }
next(status);
};
@@ -292,13 +324,22 @@ struct BlockReaderImpl::ReadPadding : continuation::Continuation {
return;
}
- auto h = [next, this](const Status &status) {
+ auto h = [next, this](const Status &stat) {
+ Status status = stat;
if (status.ok()) {
assert(reinterpret_cast<const int &>(*bytes_transferred_) ==
parent_->chunk_padding_bytes_);
parent_->chunk_padding_bytes_ = 0;
parent_->state_ = kReadData;
}
+ if(parent_->event_handlers_) {
+ event_response event_resp = parent_->event_handlers_->call(FILE_DN_READ_EVENT, "", "", 0);
+#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
+ if (status.ok() && event_resp.response() == event_response::kTest_Error) {
+ status = Status::Error("Test error");
+ }
+#endif
+ }
next(status);
};
read_data_->Run(h);
@@ -334,11 +375,20 @@ struct BlockReaderImpl::AckRead : continuation::Continuation {
m->Push(
continuation::WriteDelimitedPBMessage(parent_->dn_, &m->state()));
- m->Run([this, next](const Status &status,
+ m->Run([this, next](const Status &stat,
const hadoop::hdfs::ClientReadStatusProto &) {
+ Status status = stat;
if (status.ok()) {
parent_->state_ = BlockReaderImpl::kFinished;
}
+ if(parent_->event_handlers_) {
+ event_response event_resp = parent_->event_handlers_->call(FILE_DN_READ_EVENT, "", "", 0);
+#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
+ if (status.ok() && event_resp.response() == event_response::kTest_Error) {
+ status = Status::Error("Test error");
+ }
+#endif
+ }
next(status);
});
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a586ccbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.h
index f9794b1..b5cbdf5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.h
@@ -93,9 +93,9 @@ class BlockReaderImpl
: public BlockReader, public std::enable_shared_from_this<BlockReaderImpl> {
public:
explicit BlockReaderImpl(const BlockReaderOptions &options, std::shared_ptr<DataNodeConnection> dn,
- CancelHandle cancel_state)
+ CancelHandle cancel_state, std::shared_ptr<LibhdfsEvents> event_handlers=nullptr)
: dn_(dn), state_(kOpen), options_(options),
- chunk_padding_bytes_(0), cancel_state_(cancel_state) {}
+ chunk_padding_bytes_(0), cancel_state_(cancel_state), event_handlers_(event_handlers.get()) {}
virtual void AsyncReadPacket(
const MutableBuffers &buffers,
@@ -152,6 +152,7 @@ private:
long long bytes_to_read_;
std::vector<char> checksum_;
CancelHandle cancel_state_;
+ LibhdfsEvents* event_handlers_;
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a586ccbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
index 8567932..be6d7bd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
@@ -274,9 +274,18 @@ void RpcConnection::HandleRpcResponse(std::shared_ptr<Response> response) {
}
Status status;
- if (h.has_exceptionclassname()) {
+ if(event_handlers_) {
+ event_response event_resp = event_handlers_->call(FS_NN_READ_EVENT, cluster_name_.c_str(), 0);
+#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
+ if (event_resp.response() == event_response::kTest_Error) {
+ status = event_resp.status();
+ }
+#endif
+ }
+
+ if (status.ok() && h.has_exceptionclassname()) {
status =
- Status::Exception(h.exceptionclassname().c_str(), h.errormsg().c_str());
+ Status::Exception(h.exceptionclassname().c_str(), h.errormsg().c_str());
}
io_service().post([req, response, status]() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a586ccbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
index 2b47ce1..330f9b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
@@ -30,6 +30,8 @@
#include <asio/read.hpp>
#include <asio/write.hpp>
+#include <system_error>
+
namespace hdfs {
template <class NextLayer>
@@ -72,8 +74,9 @@ RpcConnectionImpl<NextLayer>::RpcConnectionImpl(RpcEngine *engine)
: RpcConnection(engine),
options_(engine->options()),
next_layer_(engine->io_service()),
- connect_timer_(engine->io_service()) {
- LOG_TRACE(kRPC, << "RpcConnectionImpl::RpcConnectionImpl called");
+ connect_timer_(engine->io_service())
+{
+ LOG_TRACE(kRPC, << "RpcConnectionImpl::RpcConnectionImpl called &" << (void*)this);
}
template <class NextLayer>
@@ -87,7 +90,6 @@ RpcConnectionImpl<NextLayer>::~RpcConnectionImpl() {
LOG_WARN(kRPC, << "RpcConnectionImpl::~RpcConnectionImpl called with items in the requests_on_fly queue");
}
-
template <class NextLayer>
void RpcConnectionImpl<NextLayer>::Connect(
const std::vector<::asio::ip::tcp::endpoint> &server,
@@ -171,8 +173,8 @@ void RpcConnectionImpl<NextLayer>::ConnectComplete(const ::asio::error_code &ec,
Status status = ToStatus(ec);
if(event_handlers_) {
- auto event_resp = event_handlers_->call(FS_NN_CONNECT_EVENT, cluster_name_.c_str(), 0);
-#ifndef NDEBUG
+ event_response event_resp = event_handlers_->call(FS_NN_CONNECT_EVENT, cluster_name_.c_str(), 0);
+#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
if (event_resp.response() == event_response::kTest_Error) {
status = event_resp.status();
}
@@ -349,27 +351,28 @@ void RpcConnectionImpl<NextLayer>::FlushPendingRequests() {
template <class NextLayer>
-void RpcConnectionImpl<NextLayer>::OnRecvCompleted(const ::asio::error_code &asio_ec,
+void RpcConnectionImpl<NextLayer>::OnRecvCompleted(const ::asio::error_code &original_ec,
size_t) {
using std::placeholders::_1;
using std::placeholders::_2;
std::lock_guard<std::mutex> state_lock(connection_state_lock_);
+ ::asio::error_code my_ec(original_ec);
+
LOG_TRACE(kRPC, << "RpcConnectionImpl::OnRecvCompleted called");
std::shared_ptr<RpcConnection> shared_this = shared_from_this();
- ::asio::error_code ec = asio_ec;
if(event_handlers_) {
- auto event_resp = event_handlers_->call(FS_NN_READ_EVENT, cluster_name_.c_str(), 0);
-#ifndef NDEBUG
+ event_response event_resp = event_handlers_->call(FS_NN_READ_EVENT, cluster_name_.c_str(), 0);
+#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
if (event_resp.response() == event_response::kTest_Error) {
- ec = std::make_error_code(std::errc::network_down);
+ my_ec = std::make_error_code(std::errc::network_down);
}
#endif
}
- switch (ec.value()) {
+ switch (my_ec.value()) {
case 0:
// No errors
break;
@@ -377,8 +380,8 @@ void RpcConnectionImpl<NextLayer>::OnRecvCompleted(const ::asio::error_code &asi
// The event loop has been shut down. Ignore the error.
return;
default:
- LOG_WARN(kRPC, << "Network error during RPC read: " << ec.message());
- CommsError(ToStatus(ec));
+ LOG_WARN(kRPC, << "Network error during RPC read: " << my_ec.message());
+ CommsError(ToStatus(my_ec));
return;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a586ccbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
index d0365c3..5de7d53 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
@@ -230,7 +230,6 @@ class RpcConnection : public std::enable_shared_from_this<RpcConnection> {
std::shared_ptr<LibhdfsEvents> event_handlers_;
std::string cluster_name_;
-
// Lock for mutable parts of this class that need to be thread safe
std::mutex connection_state_lock_;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a586ccbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
index b30afb9..032dfc8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
@@ -22,6 +22,9 @@ set (LIBHDFS_TESTS_DIR ../../libhdfs-tests)
set (LIBHDFSPP_SRC_DIR ..)
set (LIBHDFSPP_LIB_DIR ${LIBHDFSPP_SRC_DIR}/lib)
set (LIBHDFSPP_BINDING_C ${LIBHDFSPP_LIB_DIR}/bindings/c)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-missing-field-initializers")
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-field-initializers")
+
include_directories(
${GENERATED_JAVAH}
${CMAKE_CURRENT_LIST_DIR}
@@ -138,6 +141,10 @@ build_libhdfs_test(hdfspp_mini_dfs_smoke hdfspp_test_shim_static ${CMAKE_CURRENT
link_libhdfs_test (hdfspp_mini_dfs_smoke hdfspp_test_shim_static fs reader rpc proto common connection gmock_main ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} native_mini_dfs ${JAVA_JVM_LIBRARY} ${SASL_LIBRARIES})
add_libhdfs_test (hdfspp_mini_dfs_smoke hdfspp_test_shim_static)
+build_libhdfs_test(libhdfs_mini_stress hdfspp_test_shim_static expect.c test_libhdfs_mini_stress.c ${OS_DIR}/thread.c)
+link_libhdfs_test(libhdfs_mini_stress hdfspp_test_shim_static fs reader rpc proto common connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} native_mini_dfs ${JAVA_JVM_LIBRARY} ${SASL_LIBRARIES})
+add_libhdfs_test(libhdfs_mini_stress hdfspp_test_shim_static)
+
build_libhdfs_test(hdfs_ext hdfspp_test_shim_static ${CMAKE_CURRENT_LIST_DIR}/hdfs_ext_test.cc)
link_libhdfs_test (hdfs_ext hdfspp_test_shim_static hdfspp_static gmock_main native_mini_dfs ${JAVA_JVM_LIBRARY} ${SASL_LIBRARIES})
add_libhdfs_test (hdfs_ext hdfspp_test_shim_static)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a586ccbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc
index 01d723f..9e3aeb7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc
@@ -93,9 +93,10 @@ public:
std::shared_ptr<MockReader> mock_reader_ = std::make_shared<MockReader>();
protected:
std::shared_ptr<BlockReader> CreateBlockReader(const BlockReaderOptions &options,
- std::shared_ptr<DataNodeConnection> dn) override
+ std::shared_ptr<DataNodeConnection> dn,
+ std::shared_ptr<hdfs::LibhdfsEvents> event_handlers) override
{
- (void) options; (void) dn;
+ (void) options; (void) dn; (void) event_handlers;
assert(mock_reader_);
return mock_reader_;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a586ccbc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
index b5f4d9a..defe95d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
@@ -386,7 +386,7 @@ TEST(RpcEngineTest, TestEventCallbacks)
});
io_service.run();
ASSERT_TRUE(complete);
- ASSERT_EQ(7, callbacks.size());
+ ASSERT_EQ(8, callbacks.size());
ASSERT_EQ(FS_NN_CONNECT_EVENT, callbacks[0]); // error
ASSERT_EQ(FS_NN_CONNECT_EVENT, callbacks[1]); // reconnect
ASSERT_EQ(FS_NN_READ_EVENT, callbacks[2]); // makes an error
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/52] [abbrv] hadoop git commit: HDFS-10796: libhdfs++: Previous
commit was missing test/hdfs_ioservice_test.cc, adding it.
Posted by jh...@apache.org.
HDFS-10796: libhdfs++: Previous commit was missing test/hdfs_ioservice_test.cc, adding it.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a30cf6a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a30cf6a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a30cf6a3
Branch: refs/heads/HDFS-8707
Commit: a30cf6a369770381df4b3fb23d059749dbf6c4cf
Parents: fbff671
Author: James <jh...@apache.org>
Authored: Sun Dec 11 10:52:39 2016 -0500
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../libhdfspp/tests/hdfs_ioservice_test.cc | 103 +++++++++++++++++++
1 file changed, 103 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a30cf6a3/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ioservice_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ioservice_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ioservice_test.cc
new file mode 100644
index 0000000..a03f275
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ioservice_test.cc
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common/hdfs_ioservice.h"
+
+#include <future>
+#include <functional>
+#include <thread>
+#include <string>
+
+#include <gmock/gmock.h>
+
+using ::testing::_;
+using ::testing::InvokeArgument;
+using ::testing::Return;
+
+using namespace hdfs;
+
+// Make sure IoService spins up specified number of threads
+TEST(IoServiceTest, InitThreads) {
+#ifndef DISABLE_CONCURRENT_WORKERS
+ std::shared_ptr<IoServiceImpl> service = std::static_pointer_cast<IoServiceImpl>(IoService::MakeShared());
+ EXPECT_NE(service, nullptr);
+
+ unsigned int thread_count = 4;
+ unsigned int result_thread_count = service->InitWorkers(thread_count);
+ EXPECT_EQ(thread_count, result_thread_count);
+
+ service->Stop();
+#else
+ #pragma message("DISABLE_CONCURRENT_WORKERS is defined so hdfs_ioservice_test will compile out the InitThreads test")
+#endif
+}
+
+// Make sure IoService defaults to logical thread count
+TEST(IoServiceTest, InitDefaultThreads) {
+#ifndef DISABLE_CONCURRENT_WORKERS
+ std::shared_ptr<IoServiceImpl> service = std::static_pointer_cast<IoServiceImpl>(IoService::MakeShared());
+ EXPECT_NE(service, nullptr);
+
+ unsigned int thread_count = std::thread::hardware_concurrency();
+ unsigned int result_thread_count = service->InitDefaultWorkers();
+ EXPECT_EQ(thread_count, result_thread_count);
+
+ service->Stop();
+#else
+ #pragma message("DISABLE_CONCURRENT_WORKERS is defined so hdfs_ioservice_test will compile out the InitDefaultThreads test")
+#endif
+}
+
+
+// Check IoService::PostTask
+TEST(IoServiceTest, SimplePost) {
+ std::shared_ptr<IoServiceImpl> service = std::static_pointer_cast<IoServiceImpl>(IoService::MakeShared());
+ EXPECT_NE(service, nullptr);
+
+ unsigned int thread_count = std::thread::hardware_concurrency();
+ unsigned int result_thread_count = service->InitDefaultWorkers();
+#ifndef DISABLE_CONCURRENT_WORKERS
+ EXPECT_EQ(thread_count, result_thread_count);
+#else
+ (void)thread_count;
+ (void)result_thread_count;
+#endif
+ // Like with the C synchronous shims a promise/future is needed to block until the async call completes.
+ auto promise = std::make_shared<std::promise<std::string>>();
+ std::future<std::string> future = promise->get_future();
+
+ // this will get invoked on a worker thread
+ std::function<void()> example_callback = [promise](){
+ promise->set_value("hello from IoService");
+ };
+ service->PostTask(example_callback);
+
+ // block until worker thread finishes
+ std::string result = future.get();
+ EXPECT_EQ(result, "hello from IoService");
+
+ service->Stop();
+
+}
+
+int main(int argc, char *argv[]) {
+ // The following line must be executed to initialize Google Mock
+ // (and Google Test) before running the tests.
+ ::testing::InitGoogleMock(&argc, argv);
+ return RUN_ALL_TESTS();
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/52] [abbrv] hadoop git commit: HDFS-10785: libhdfs++: Implement
the rest of the tools. Contributed by Anatoli Schein
Posted by jh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cpp
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cpp b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cpp
deleted file mode 100644
index 08724c6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cpp
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include <future>
-#include "tools_common.h"
-
-void usage(){
- std::cout << "Usage: hdfs_chown [OPTION] [OWNER][:[GROUP]] FILE"
- << std::endl
- << std::endl << "Change the owner and/or group of each FILE to OWNER and/or GROUP."
- << std::endl << "The user must be a super-user. Additional information is in the Permissions Guide:"
- << std::endl << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html"
- << std::endl
- << std::endl << " -R operate on files and directories recursively"
- << std::endl << " -h display this help and exit"
- << std::endl
- << std::endl << "Owner is unchanged if missing. Group is unchanged if missing."
- << std::endl << "OWNER and GROUP may be numeric as well as symbolic."
- << std::endl
- << std::endl << "Examples:"
- << std::endl << "hdfs_chown -R new_owner:new_group hdfs://localhost.localdomain:9433/dir/file"
- << std::endl << "hdfs_chown new_owner /dir/file"
- << std::endl;
-}
-
-struct SetOwnerState {
- const std::string username;
- const std::string groupname;
- const std::function<void(const hdfs::Status &)> handler;
- //The request counter is incremented once every time SetOwner async call is made
- uint64_t request_counter;
- //This boolean will be set when find returns the last result
- bool find_is_done;
- //Final status to be returned
- hdfs::Status status;
- //Shared variables will need protection with a lock
- std::mutex lock;
- SetOwnerState(const std::string & username_, const std::string & groupname_,
- const std::function<void(const hdfs::Status &)> & handler_,
- uint64_t request_counter_, bool find_is_done_)
- : username(username_),
- groupname(groupname_),
- handler(handler_),
- request_counter(request_counter_),
- find_is_done(find_is_done_),
- status(),
- lock() {
- }
-};
-
-int main(int argc, char *argv[]) {
- //We should have 3 or 4 parameters
- if (argc != 3 && argc != 4) {
- usage();
- exit(EXIT_FAILURE);
- }
-
- bool recursive = false;
- int input;
-
- //Using GetOpt to read in the values
- opterr = 0;
- while ((input = getopt(argc, argv, "Rh")) != -1) {
- switch (input)
- {
- case 'R':
- recursive = 1;
- break;
- case 'h':
- usage();
- exit(EXIT_SUCCESS);
- break;
- case '?':
- if (isprint(optopt))
- std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
- else
- std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
- usage();
- exit(EXIT_FAILURE);
- default:
- exit(EXIT_FAILURE);
- }
- }
- std::string owner_and_group = argv[optind];
- std::string uri_path = argv[optind + 1];
-
- std::string owner, group;
- size_t owner_end = owner_and_group.find(":");
- if(owner_end == std::string::npos) {
- owner = owner_and_group;
- } else {
- owner = owner_and_group.substr(0, owner_end);
- group = owner_and_group.substr(owner_end + 1);
- }
-
- //Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
-
- //TODO: HDFS-9539 Currently options can be returned empty
- hdfs::Options options = *hdfs::getOptions();
-
- //TODO: HDFS-9539 - until then we increase the time-out to allow all recursive async calls to finish
- options.rpc_timeout = std::numeric_limits<int>::max();
-
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), options);
- if (!fs) {
- std::cerr << "Could not connect the file system. " << std::endl;
- exit(EXIT_FAILURE);
- }
-
- /* wrap async FileSystem::SetOwner with promise to make it a blocking call */
- std::shared_ptr<std::promise<hdfs::Status>> promise = std::make_shared<std::promise<hdfs::Status>>();
- std::future<hdfs::Status> future(promise->get_future());
- auto handler = [promise](const hdfs::Status &s) {
- promise->set_value(s);
- };
-
- if(!recursive){
- fs->SetOwner(uri->get_path(), owner, group, handler);
- }
- else {
- //Allocating shared state, which includes:
- //username and groupname to be set, handler to be called, request counter, and a boolean to keep track if find is done
- std::shared_ptr<SetOwnerState> state = std::make_shared<SetOwnerState>(owner, group, handler, 0, false);
-
- // Keep requesting more from Find until we process the entire listing. Call handler when Find is done and reques counter is 0.
- // Find guarantees that the handler will only be called once at a time so we do not need locking in handlerFind.
- auto handlerFind = [fs, state](const hdfs::Status &status_find, const std::vector<hdfs::StatInfo> & stat_infos, bool has_more_results) -> bool {
-
- //For each result returned by Find we call async SetOwner with the handler below.
- //SetOwner DOES NOT guarantee that the handler will only be called once at a time, so we DO need locking in handlerSetOwner.
- auto handlerSetOwner = [state](const hdfs::Status &status_set_owner) {
- std::lock_guard<std::mutex> guard(state->lock);
-
- //Decrement the counter once since we are done with this async call
- if (!status_set_owner.ok() && state->status.ok()){
- //We make sure we set state->status only on the first error.
- state->status = status_set_owner;
- }
- state->request_counter--;
- if(state->request_counter == 0 && state->find_is_done){
- state->handler(state->status); //exit
- }
- };
- if(!stat_infos.empty() && state->status.ok()) {
- for (hdfs::StatInfo const& s : stat_infos) {
- //Launch an asynchronous call to SetOwner for every returned result
- state->request_counter++;
- fs->SetOwner(s.full_path, state->username, state->groupname, handlerSetOwner);
- }
- }
-
- //Lock this section because handlerSetOwner might be accessing the same
- //shared variables simultaneously
- std::lock_guard<std::mutex> guard(state->lock);
- if (!status_find.ok() && state->status.ok()){
- //We make sure we set state->status only on the first error.
- state->status = status_find;
- }
- if(!has_more_results){
- state->find_is_done = true;
- if(state->request_counter == 0){
- state->handler(state->status); //exit
- }
- return false;
- }
- return true;
- };
-
- //Asynchronous call to Find
- fs->Find(uri->get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
- }
-
- /* block until promise is set */
- hdfs::Status status = future.get();
- if (!status.ok()) {
- std::cerr << "Error: " << status.ToString() << std::endl;
- exit(EXIT_FAILURE);
- }
-
- // Clean up static data and prevent valgrind memory leaks
- google::protobuf::ShutdownProtobufLibrary();
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_copyToLocal.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_copyToLocal.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_copyToLocal.cc
new file mode 100644
index 0000000..493ff62
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_copyToLocal.cc
@@ -0,0 +1,92 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_copyToLocal [OPTION] SRC_FILE DST_FILE"
+ << std::endl
+ << std::endl << "Copy SRC_FILE from hdfs to DST_FILE on the local file system."
+ << std::endl
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_copyToLocal hdfs://localhost.localdomain:8020/dir/file /home/usr/myfile"
+ << std::endl << "hdfs_copyToLocal /dir/file /home/usr/dir/file"
+ << std::endl;
+}
+
+int main(int argc, char *argv[]) {
+ if (argc > 4) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ int input;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "h")) != -1) {
+ switch (input)
+ {
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ case '?':
+ if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ std::string uri_path = argv[optind];
+ std::string dest = argv[optind+1];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::FILE* dst_file = std::fopen(dest.c_str(), "wb");
+ if(!dst_file){
+ std::cerr << "Unable to open the destination file: " << dest << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ readFile(fs, uri->get_path(), 0, dst_file, false);
+ std::fclose(dst_file);
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_count.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_count.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_count.cc
new file mode 100644
index 0000000..e43596c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_count.cc
@@ -0,0 +1,97 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_count [OPTION] FILE"
+ << std::endl
+ << std::endl << "Count the number of directories, files and bytes under the path that match the specified FILE pattern."
+ << std::endl << "The output columns with -count are: DIR_COUNT, FILE_COUNT, CONTENT_SIZE, PATHNAME"
+ << std::endl
+ << std::endl << " -q output additional columns before the rest: QUOTA, SPACE_QUOTA, SPACE_CONSUMED"
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_count hdfs://localhost.localdomain:8020/dir"
+ << std::endl << "hdfs_count -q /dir1/dir2"
+ << std::endl;
+}
+
+int main(int argc, char *argv[]) {
+ //We should have at least 2 arguments
+ if (argc < 2) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ bool quota = false;
+ int input;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "qh")) != -1) {
+ switch (input)
+ {
+ case 'q':
+ quota = true;
+ break;
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ case '?':
+ if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+ std::string uri_path = argv[optind];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ hdfs::ContentSummary content_summary;
+ hdfs::Status status = fs->GetContentSummary(uri->get_path(), content_summary);
+ if (!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ std::cout << content_summary.str(quota) << std::endl;
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_createSnapshot.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_createSnapshot.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_createSnapshot.cc
new file mode 100644
index 0000000..fd079a2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_createSnapshot.cc
@@ -0,0 +1,99 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_createSnapshot [OPTION] PATH"
+ << std::endl
+ << std::endl << "Create a snapshot of a snapshottable directory."
+ << std::endl << "This operation requires owner privilege of the snapshottable directory."
+ << std::endl
+ << std::endl << " -n NAME The snapshot name. When it is omitted, a default name is generated"
+ << std::endl << " using a timestamp with the format:"
+ << std::endl << " \"'s'yyyyMMdd-HHmmss.SSS\", e.g. s20130412-151029.033"
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_createSnapshot hdfs://localhost.localdomain:8020/dir"
+ << std::endl << "hdfs_createSnapshot -n MySnapshot /dir1/dir2"
+ << std::endl;
+}
+
+int main(int argc, char *argv[]) {
+ //We should have at least 2 arguments
+ if (argc < 2) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ int input;
+ std::string name;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "hn:")) != -1) {
+ switch (input)
+ {
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ case 'n':
+ name = optarg;
+ break;
+ case '?':
+ if (optopt == 'n')
+ std::cerr << "Option -" << (char) optopt << " requires an argument." << std::endl;
+ else if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+ std::string uri_path = argv[optind];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ hdfs::Status status = fs->CreateSnapshot(uri->get_path(), name);
+ if (!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_deleteSnapshot.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_deleteSnapshot.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_deleteSnapshot.cc
new file mode 100644
index 0000000..2082878
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_deleteSnapshot.cc
@@ -0,0 +1,91 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_deleteSnapshot [OPTION] PATH NAME"
+ << std::endl
+ << std::endl << "Delete a snapshot NAME from a snapshottable directory."
+ << std::endl << "This operation requires owner privilege of the snapshottable directory."
+ << std::endl
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_deleteSnapshot hdfs://localhost.localdomain:8020/dir mySnapshot"
+ << std::endl << "hdfs_deleteSnapshot /dir1/dir2 mySnapshot"
+ << std::endl;
+}
+
+int main(int argc, char *argv[]) {
+ //We should have at least 2 arguments
+ if (argc < 2) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ int input;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "h")) != -1) {
+ switch (input)
+ {
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ case '?':
+ if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+ std::string uri_path = argv[optind];
+ std::string name = argv[optind+1];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ hdfs::Status status = fs->DeleteSnapshot(uri->get_path(), name);
+ if (!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_df.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_df.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_df.cc
new file mode 100644
index 0000000..6170e93
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_df.cc
@@ -0,0 +1,93 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_df [OPTION] PATH"
+ << std::endl
+ << std::endl << "Displays size, used space, and available space of"
+ << std::endl << "the entire filesystem where PATH is located"
+ << std::endl
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_df hdfs://localhost.localdomain:8020/"
+ << std::endl << "hdfs_df /"
+ << std::endl;
+}
+
+int main(int argc, char *argv[]) {
+ //We should have at least 2 arguments
+ if (argc < 2) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ int input;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "h")) != -1) {
+ switch (input)
+ {
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ case '?':
+ if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+ std::string uri_path = argv[optind];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ hdfs::FsInfo fs_info;
+
+ hdfs::Status status = fs->GetFsStats(fs_info);
+ if (!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ std::cout << fs_info.str("hdfs://" + fs->get_cluster_name()) << std::endl;
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_disallowSnapshot.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_disallowSnapshot.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_disallowSnapshot.cc
new file mode 100644
index 0000000..c181719
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_disallowSnapshot.cc
@@ -0,0 +1,90 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_disallowSnapshot [OPTION] PATH"
+ << std::endl
+ << std::endl << "Disallowing snapshots of a directory at PATH to be created."
+ << std::endl << "All snapshots of the directory must be deleted before disallowing snapshots."
+ << std::endl
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_disallowSnapshot hdfs://localhost.localdomain:8020/dir"
+ << std::endl << "hdfs_disallowSnapshot /dir1/dir2"
+ << std::endl;
+}
+
+int main(int argc, char *argv[]) {
+ //We should have at least 2 arguments
+ if (argc < 2) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ int input;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "Rh")) != -1) {
+ switch (input)
+ {
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ case '?':
+ if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+ std::string uri_path = argv[optind];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ hdfs::Status status = fs->DisallowSnapshot(uri->get_path());
+ if (!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_du.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_du.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_du.cc
new file mode 100644
index 0000000..30269e3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_du.cc
@@ -0,0 +1,180 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include <future>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_du [OPTION] PATH"
+ << std::endl
+ << std::endl << "Displays sizes of files and directories contained in the given PATH"
+ << std::endl << "or the length of a file in case PATH is just a file"
+ << std::endl
+ << std::endl << " -R operate on files and directories recursively"
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_du hdfs://localhost.localdomain:8020/dir/file"
+ << std::endl << "hdfs_du -R /dir1/dir2"
+ << std::endl;
+}
+
+struct GetContentSummaryState {
+ const std::function<void(const hdfs::Status &)> handler;
+ //The request counter is incremented once every time GetContentSummary async call is made
+ uint64_t request_counter;
+ //This boolean will be set when find returns the last result
+ bool find_is_done;
+ //Final status to be returned
+ hdfs::Status status;
+ //Shared variables will need protection with a lock
+ std::mutex lock;
+ GetContentSummaryState(const std::function<void(const hdfs::Status &)> & handler_,
+ uint64_t request_counter_, bool find_is_done_)
+ : handler(handler_),
+ request_counter(request_counter_),
+ find_is_done(find_is_done_),
+ status(),
+ lock() {
+ }
+};
+
+int main(int argc, char *argv[]) {
+ //We should have at least 2 arguments
+ if (argc < 2) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ bool recursive = false;
+ int input;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "Rh")) != -1) {
+ switch (input)
+ {
+ case 'R':
+ recursive = true;
+ break;
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ case '?':
+ if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+ std::string uri_path = argv[optind];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), true);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ /* wrap async FileSystem::GetContentSummary with promise to make it a blocking call */
+ std::shared_ptr<std::promise<hdfs::Status>> promise = std::make_shared<std::promise<hdfs::Status>>();
+ std::future<hdfs::Status> future(promise->get_future());
+ auto handler = [promise](const hdfs::Status &s) {
+ promise->set_value(s);
+ };
+
+ //Allocating shared state, which includes:
+ //handler to be called, request counter, and a boolean to keep track if find is done
+ std::shared_ptr<GetContentSummaryState> state = std::make_shared<GetContentSummaryState>(handler, 0, false);
+
+ // Keep requesting more from Find until we process the entire listing. Call handler when Find is done and reques counter is 0.
+ // Find guarantees that the handler will only be called once at a time so we do not need locking in handlerFind.
+ auto handlerFind = [fs, state](const hdfs::Status &status_find, const std::vector<hdfs::StatInfo> & stat_infos, bool has_more_results) -> bool {
+
+ //For each result returned by Find we call async GetContentSummary with the handler below.
+ //GetContentSummary DOES NOT guarantee that the handler will only be called once at a time, so we DO need locking in handlerGetContentSummary.
+ auto handlerGetContentSummary = [state](const hdfs::Status &status_get_summary, const hdfs::ContentSummary &si) {
+ std::lock_guard<std::mutex> guard(state->lock);
+ std::cout << si.str_du() << std::endl;
+ //Decrement the counter once since we are done with this async call
+ if (!status_get_summary.ok() && state->status.ok()){
+ //We make sure we set state->status only on the first error.
+ state->status = status_get_summary;
+ }
+ state->request_counter--;
+ if(state->request_counter == 0 && state->find_is_done){
+ state->handler(state->status); //exit
+ }
+ };
+ if(!stat_infos.empty() && state->status.ok()) {
+ for (hdfs::StatInfo const& s : stat_infos) {
+ //Launch an asynchronous call to GetContentSummary for every returned result
+ state->request_counter++;
+ fs->GetContentSummary(s.full_path, handlerGetContentSummary);
+ }
+ }
+
+ //Lock this section because handlerGetContentSummary might be accessing the same
+ //shared variables simultaneously
+ std::lock_guard<std::mutex> guard(state->lock);
+ if (!status_find.ok() && state->status.ok()){
+ //We make sure we set state->status only on the first error.
+ state->status = status_find;
+ }
+ if(!has_more_results){
+ state->find_is_done = true;
+ if(state->request_counter == 0){
+ state->handler(state->status); //exit
+ }
+ return false;
+ }
+ return true;
+ };
+
+ if(!recursive){
+ //Asynchronous call to Find
+ fs->GetListing(uri->get_path(), handlerFind);
+ } else {
+ //Asynchronous call to Find
+ fs->Find(uri->get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
+ }
+
+ /* block until promise is set */
+ hdfs::Status status = future.get();
+ if (!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_find.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_find.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_find.cc
new file mode 100644
index 0000000..ef2ad7b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_find.cc
@@ -0,0 +1,150 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include <future>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_find [OPTION] PATH"
+ << std::endl
+ << std::endl << "Finds all files recursively starting from the"
+ << std::endl << "specified PATH and prints their file paths."
+ << std::endl << "This hdfs_find tool mimics the POSIX find."
+ << std::endl
+ << std::endl << "Both PATH and NAME can have wild-cards."
+ << std::endl
+ << std::endl << " -n NAME if provided all results will be matching the NAME pattern"
+ << std::endl << " otherwise, the implicit '*' will be used"
+ << std::endl << " NAME allows wild-cards"
+ << std::endl
+ << std::endl << " -m MAX_DEPTH if provided the maximum depth to recurse after the end of"
+ << std::endl << " the path is reached will be limited by MAX_DEPTH"
+ << std::endl << " otherwise, the maximum depth to recurse is unbound"
+ << std::endl << " MAX_DEPTH can be set to 0 for pure globbing and ignoring"
+ << std::endl << " the NAME option (no recursion after the end of the path)"
+ << std::endl
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_find hdfs://localhost.localdomain:8020/dir?/tree* -n some?file*name"
+ << std::endl << "hdfs_find / -n file_name -m 3"
+ << std::endl;
+}
+
+int main(int argc, char *argv[]) {
+ //We should have at least 2 arguments
+ if (argc < 2) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ int input;
+ //If NAME is not specified we use implicit "*"
+ std::string name = "*";
+ //If MAX_DEPTH is not specified we use the max value of uint_32_t
+ uint32_t max_depth = hdfs::FileSystem::GetDefaultFindMaxDepth();
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "hn:m:")) != -1) {
+ switch (input)
+ {
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ case 'n':
+ name = optarg;
+ break;
+ case 'm':
+ max_depth = std::stoi(optarg);
+ break;
+ case '?':
+ if (optopt == 'n' || optopt == 'm')
+ std::cerr << "Option -" << (char) optopt << " requires an argument." << std::endl;
+ else if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+ std::string uri_path = argv[optind];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), true);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::shared_ptr<std::promise<void>> promise = std::make_shared<std::promise<void>>();
+ std::future<void> future(promise->get_future());
+ hdfs::Status status = hdfs::Status::OK();
+
+ /**
+ * Keep requesting more until we get the entire listing. Set the promise
+ * when we have the entire listing to stop.
+ *
+ * Find guarantees that the handler will only be called once at a time,
+ * so we do not need any locking here. It also guarantees that the handler will be
+ * only called once with has_more_results set to false.
+ */
+ auto handler = [promise, &status]
+ (const hdfs::Status &s, const std::vector<hdfs::StatInfo> & si, bool has_more_results) -> bool {
+ //Print result chunks as they arrive
+ if(!si.empty()) {
+ for (hdfs::StatInfo const& s : si) {
+ std::cout << s.str() << std::endl;
+ }
+ }
+ if(!s.ok() && status.ok()){
+ //We make sure we set 'status' only on the first error.
+ status = s;
+ }
+ if (!has_more_results) {
+ promise->set_value(); //set promise
+ return false; //request stop sending results
+ }
+ return true; //request more results
+ };
+
+ //Asynchronous call to Find
+ fs->Find(uri->get_path(), name, max_depth, handler);
+
+ //block until promise is set
+ future.get();
+ if(!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ }
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_find.cpp
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_find.cpp b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_find.cpp
deleted file mode 100644
index eca79c6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_find.cpp
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include <future>
-#include "tools_common.h"
-
-void usage(){
- std::cout << "Usage: hdfs_find [OPTION] PATH"
- << std::endl
- << std::endl << "Finds all files recursively starting from the"
- << std::endl << "specified PATH and prints their file paths."
- << std::endl << "This hdfs_find tool mimics the POSIX find."
- << std::endl
- << std::endl << "Both PATH and NAME can have wild-cards."
- << std::endl
- << std::endl << " -n NAME if provided all results will be matching the NAME pattern"
- << std::endl << " otherwise, the implicit '*' will be used"
- << std::endl << " NAME allows wild-cards"
- << std::endl
- << std::endl << " -m MAX_DEPTH if provided the maximum depth to recurse after the end of"
- << std::endl << " the path is reached will be limited by MAX_DEPTH"
- << std::endl << " otherwise, the maximum depth to recurse is unbound"
- << std::endl << " MAX_DEPTH can be set to 0 for pure globbing and ignoring"
- << std::endl << " the NAME option (no recursion after the end of the path)"
- << std::endl
- << std::endl << " -h display this help and exit"
- << std::endl
- << std::endl << "Examples:"
- << std::endl << "hdfs_find hdfs://localhost.localdomain:9433/dir?/tree* -n some?file*name"
- << std::endl << "hdfs_find / -n file_name -m 3"
- << std::endl;
-}
-
-int main(int argc, char *argv[]) {
- //We should have at least 2 arguments
- if (argc < 2) {
- usage();
- exit(EXIT_FAILURE);
- }
-
- int input;
- //If NAME is not specified we use implicit "*"
- std::string name = "*";
- //If MAX_DEPTH is not specified we use the max value of uint_32_t
- uint32_t max_depth = hdfs::FileSystem::GetDefaultFindMaxDepth();
-
- //Using GetOpt to read in the values
- opterr = 0;
- while ((input = getopt(argc, argv, "hn:m:")) != -1) {
- switch (input)
- {
- case 'h':
- usage();
- exit(EXIT_SUCCESS);
- break;
- case 'n':
- name = optarg;
- break;
- case 'm':
- max_depth = std::stoi(optarg);
- break;
- case '?':
- if (optopt == 'n' || optopt == 'm')
- std::cerr << "Option -" << (char) optopt << " requires an argument." << std::endl;
- else if (isprint(optopt))
- std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
- else
- std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
- usage();
- exit(EXIT_FAILURE);
- default:
- exit(EXIT_FAILURE);
- }
- }
- std::string uri_path = argv[optind];
-
- //Building a URI object from the given uri_path
- hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
- if (!uri) {
- std::cerr << "Malformed URI: " << uri_path << std::endl;
- exit(EXIT_FAILURE);
- }
-
- //TODO: HDFS-9539 Currently options can be returned empty
- hdfs::Options options = *hdfs::getOptions();
-
- //TODO: HDFS-9539 - until then we increase the time-out to allow all recursive async calls to finish
- options.rpc_timeout = std::numeric_limits<int>::max();
-
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), options);
- if (!fs) {
- std::cerr << "Could not connect the file system. " << std::endl;
- exit(EXIT_FAILURE);
- }
-
- std::promise<void> promise;
- std::future<void> future(promise.get_future());
- hdfs::Status status = hdfs::Status::OK();
-
- /**
- * Keep requesting more until we get the entire listing. Set the promise
- * when we have the entire listing to stop.
- *
- * Find guarantees that the handler will only be called once at a time,
- * so we do not need any locking here
- */
- auto handler = [&promise, &status]
- (const hdfs::Status &s, const std::vector<hdfs::StatInfo> & si, bool has_more_results) -> bool {
- //Print result chunks as they arrive
- if(!si.empty()) {
- for (hdfs::StatInfo const& s : si) {
- std::cout << s.full_path << std::endl;
- }
- }
- if(!s.ok() && status.ok()){
- //We make sure we set 'status' only on the first error.
- status = s;
- }
- if (!has_more_results) {
- promise.set_value(); //set promise
- return false; //request stop sending results
- }
- return true; //request more results
- };
-
- //Asynchronous call to Find
- fs->Find(uri->get_path(), name, max_depth, handler);
-
- //block until promise is set
- future.get();
- if(!status.ok()) {
- std::cerr << "Error: " << status.ToString() << std::endl;
- }
-
- // Clean up static data and prevent valgrind memory leaks
- google::protobuf::ShutdownProtobufLibrary();
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_get.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_get.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_get.cc
new file mode 100644
index 0000000..f1ff1c8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_get.cc
@@ -0,0 +1,92 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_get [OPTION] SRC_FILE DST_FILE"
+ << std::endl
+ << std::endl << "Copy SRC_FILE from hdfs to DST_FILE on the local file system."
+ << std::endl
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_get hdfs://localhost.localdomain:8020/dir/file /home/usr/myfile"
+ << std::endl << "hdfs_get /dir/file /home/usr/dir/file"
+ << std::endl;
+}
+
+int main(int argc, char *argv[]) {
+ if (argc > 4) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ int input;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "h")) != -1) {
+ switch (input)
+ {
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ case '?':
+ if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ std::string uri_path = argv[optind];
+ std::string dest = argv[optind+1];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::FILE* dst_file = std::fopen(dest.c_str(), "wb");
+ if(!dst_file){
+ std::cerr << "Unable to open the destination file: " << dest << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ readFile(fs, uri->get_path(), 0, dst_file, false);
+ std::fclose(dst_file);
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_ls.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_ls.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_ls.cc
new file mode 100644
index 0000000..710dbf7
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_ls.cc
@@ -0,0 +1,134 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include <future>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_ls [OPTION] FILE"
+ << std::endl
+ << std::endl << "List information about the FILEs."
+ << std::endl
+ << std::endl << " -R list subdirectories recursively"
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_ls hdfs://localhost.localdomain:8020/dir"
+ << std::endl << "hdfs_ls -R /dir1/dir2"
+ << std::endl;
+}
+
+int main(int argc, char *argv[]) {
+ //We should have at least 2 arguments
+ if (argc < 2) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ bool recursive = false;
+ int input;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "Rh")) != -1) {
+ switch (input)
+ {
+ case 'R':
+ recursive = true;
+ break;
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ case '?':
+ if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+ std::string uri_path = argv[optind];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), true);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::shared_ptr<std::promise<void>> promise = std::make_shared<std::promise<void>>();
+ std::future<void> future(promise->get_future());
+ hdfs::Status status = hdfs::Status::OK();
+
+ /**
+ * Keep requesting more until we get the entire listing. Set the promise
+ * when we have the entire listing to stop.
+ *
+ * Find and GetListing guarantee that the handler will only be called once at a time,
+ * so we do not need any locking here. They also guarantee that the handler will be
+ * only called once with has_more_results set to false.
+ */
+ auto handler = [promise, &status]
+ (const hdfs::Status &s, const std::vector<hdfs::StatInfo> & si, bool has_more_results) -> bool {
+ //Print result chunks as they arrive
+ if(!si.empty()) {
+ for (hdfs::StatInfo const& s : si) {
+ std::cout << s.str() << std::endl;
+ }
+ }
+ if(!s.ok() && status.ok()){
+ //We make sure we set 'status' only on the first error.
+ status = s;
+ }
+ if (!has_more_results) {
+ promise->set_value(); //set promise
+ return false; //request stop sending results
+ }
+ return true; //request more results
+ };
+
+ if(!recursive){
+ //Asynchronous call to GetListing
+ fs->GetListing(uri->get_path(), handler);
+ } else {
+ //Asynchronous call to Find
+ fs->Find(uri->get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handler);
+ }
+
+ //block until promise is set
+ future.get();
+ if(!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ }
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_mkdir.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_mkdir.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_mkdir.cc
new file mode 100644
index 0000000..c4f08e3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_mkdir.cc
@@ -0,0 +1,102 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_mkdir [OPTION] DIRECTORY"
+ << std::endl
+ << std::endl << "Create the DIRECTORY(ies), if they do not already exist."
+ << std::endl
+ << std::endl << " -p make parent directories as needed"
+ << std::endl << " -m MODE set file mode (octal permissions) for the new DIRECTORY(ies)"
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_mkdir hdfs://localhost.localdomain:8020/dir1/dir2"
+ << std::endl << "hdfs_mkdir -p /extant_dir/non_extant_dir/non_extant_dir/new_dir"
+ << std::endl;
+}
+
+int main(int argc, char *argv[]) {
+ //We should have at least 2 arguments
+ if (argc < 2) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ bool create_parents = false;
+ uint16_t permissions = hdfs::FileSystem::GetDefaultPermissionMask();
+ int input;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "pm:h")) != -1) {
+ switch (input)
+ {
+ case 'p':
+ create_parents = true;
+ break;
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ case 'm':
+ //Get octal permissions for the new DIRECTORY(ies)
+ permissions = strtol(optarg, NULL, 8);
+ break;
+ case '?':
+ if (optopt == 'm')
+ std::cerr << "Option -" << (char) optopt << " requires an argument." << std::endl;
+ else if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+ std::string uri_path = argv[optind];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ hdfs::Status status = fs->Mkdirs(uri->get_path(), permissions, create_parents);
+ if (!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_moveToLocal.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_moveToLocal.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_moveToLocal.cc
new file mode 100644
index 0000000..5bb0754
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_moveToLocal.cc
@@ -0,0 +1,94 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_moveToLocal [OPTION] SRC_FILE DST_FILE"
+ << std::endl
+ << std::endl << "Move SRC_FILE from hdfs to DST_FILE on the local file system."
+ << std::endl << "Moving is done by copying SRC_FILE to DST_FILE, and then"
+ << std::endl << "deleting DST_FILE if copy succeeded."
+ << std::endl
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_moveToLocal hdfs://localhost.localdomain:8020/dir/file /home/usr/myfile"
+ << std::endl << "hdfs_moveToLocal /dir/file /home/usr/dir/file"
+ << std::endl;
+}
+
+int main(int argc, char *argv[]) {
+ if (argc > 4) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ int input;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "h")) != -1) {
+ switch (input)
+ {
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ case '?':
+ if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ std::string uri_path = argv[optind];
+ std::string dest = argv[optind+1];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::FILE* dst_file = std::fopen(dest.c_str(), "wb");
+ if(!dst_file){
+ std::cerr << "Unable to open the destination file: " << dest << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ readFile(fs, uri->get_path(), 0, dst_file, true);
+ std::fclose(dst_file);
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_renameSnapshot.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_renameSnapshot.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_renameSnapshot.cc
new file mode 100644
index 0000000..3262b34
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_renameSnapshot.cc
@@ -0,0 +1,92 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_renameSnapshot [OPTION] PATH OLD_NAME NEW_NAME"
+ << std::endl
+ << std::endl << "Rename a snapshot from OLD_NAME to NEW_NAME."
+ << std::endl << "This operation requires owner privilege of the snapshottable directory."
+ << std::endl
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_renameSnapshot hdfs://localhost.localdomain:8020/dir oldDir newDir"
+ << std::endl << "hdfs_renameSnapshot /dir1/dir2 oldSnap newSnap"
+ << std::endl;
+}
+
+int main(int argc, char *argv[]) {
+ //We should have at least 2 arguments
+ if (argc < 2) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ int input;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "h")) != -1) {
+ switch (input)
+ {
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ case '?':
+ if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+ std::string uri_path = argv[optind];
+ std::string old_name = argv[optind+1];
+ std::string new_name = argv[optind+2];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ hdfs::Status status = fs->RenameSnapshot(uri->get_path(), old_name, new_name);
+ if (!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_rm.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_rm.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_rm.cc
new file mode 100644
index 0000000..ddea730
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_rm.cc
@@ -0,0 +1,94 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_rm [OPTION] FILE"
+ << std::endl
+ << std::endl << "Remove (unlink) the FILE(s) or directory(ies)."
+ << std::endl
+ << std::endl << " -R remove directories and their contents recursively"
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_rm hdfs://localhost.localdomain:8020/dir/file"
+ << std::endl << "hdfs_rm -R /dir1/dir2"
+ << std::endl;
+}
+
+int main(int argc, char *argv[]) {
+ //We should have at least 2 arguments
+ if (argc < 2) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ bool recursive = false;
+ int input;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "Rh")) != -1) {
+ switch (input)
+ {
+ case 'R':
+ recursive = true;
+ break;
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ case '?':
+ if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+ std::string uri_path = argv[optind];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), true);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ hdfs::Status status = fs->Delete(uri->get_path(), recursive);
+ if (!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_setrep.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_setrep.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_setrep.cc
new file mode 100644
index 0000000..97aaee2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_setrep.cc
@@ -0,0 +1,176 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include <future>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_setrep [OPTION] NUM_REPLICAS PATH"
+ << std::endl
+ << std::endl << "Changes the replication factor of a file at PATH. If PATH is a directory then the command"
+ << std::endl << "recursively changes the replication factor of all files under the directory tree rooted at PATH."
+ << std::endl
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_setrep 5 hdfs://localhost.localdomain:8020/dir/file"
+ << std::endl << "hdfs_setrep 3 /dir1/dir2"
+ << std::endl;
+}
+
+struct SetReplicationState {
+ const uint16_t replication;
+ const std::function<void(const hdfs::Status &)> handler;
+ //The request counter is incremented once every time SetReplication async call is made
+ uint64_t request_counter;
+ //This boolean will be set when find returns the last result
+ bool find_is_done;
+ //Final status to be returned
+ hdfs::Status status;
+ //Shared variables will need protection with a lock
+ std::mutex lock;
+ SetReplicationState(const uint16_t replication_, const std::function<void(const hdfs::Status &)> & handler_,
+ uint64_t request_counter_, bool find_is_done_)
+ : replication(replication_),
+ handler(handler_),
+ request_counter(request_counter_),
+ find_is_done(find_is_done_),
+ status(),
+ lock() {
+ }
+};
+
+int main(int argc, char *argv[]) {
+ //We should have 3 or 4 parameters
+ if (argc < 3) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ int input;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "h")) != -1) {
+ switch (input)
+ {
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ case '?':
+ if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+ std::string repl = argv[optind];
+ std::string uri_path = argv[optind + 1];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), true);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ /* wrap async FileSystem::SetReplication with promise to make it a blocking call */
+ std::shared_ptr<std::promise<hdfs::Status>> promise = std::make_shared<std::promise<hdfs::Status>>();
+ std::future<hdfs::Status> future(promise->get_future());
+ auto handler = [promise](const hdfs::Status &s) {
+ promise->set_value(s);
+ };
+
+ uint16_t replication = std::stoi(repl.c_str(), NULL, 8);
+ //Allocating shared state, which includes:
+ //replication to be set, handler to be called, request counter, and a boolean to keep track if find is done
+ std::shared_ptr<SetReplicationState> state = std::make_shared<SetReplicationState>(replication, handler, 0, false);
+
+ // Keep requesting more from Find until we process the entire listing. Call handler when Find is done and reques counter is 0.
+ // Find guarantees that the handler will only be called once at a time so we do not need locking in handlerFind.
+ auto handlerFind = [fs, state](const hdfs::Status &status_find, const std::vector<hdfs::StatInfo> & stat_infos, bool has_more_results) -> bool {
+
+ //For each result returned by Find we call async SetReplication with the handler below.
+ //SetReplication DOES NOT guarantee that the handler will only be called once at a time, so we DO need locking in handlerSetReplication.
+ auto handlerSetReplication = [state](const hdfs::Status &status_set_replication) {
+ std::lock_guard<std::mutex> guard(state->lock);
+
+ //Decrement the counter once since we are done with this async call
+ if (!status_set_replication.ok() && state->status.ok()){
+ //We make sure we set state->status only on the first error.
+ state->status = status_set_replication;
+ }
+ state->request_counter--;
+ if(state->request_counter == 0 && state->find_is_done){
+ state->handler(state->status); //exit
+ }
+ };
+ if(!stat_infos.empty() && state->status.ok()) {
+ for (hdfs::StatInfo const& s : stat_infos) {
+ //Launch an asynchronous call to SetReplication for every returned file
+ if(s.file_type == hdfs::StatInfo::IS_FILE){
+ state->request_counter++;
+ fs->SetReplication(s.full_path, state->replication, handlerSetReplication);
+ }
+ }
+ }
+
+ //Lock this section because handlerSetReplication might be accessing the same
+ //shared variables simultaneously
+ std::lock_guard<std::mutex> guard(state->lock);
+ if (!status_find.ok() && state->status.ok()){
+ //We make sure we set state->status only on the first error.
+ state->status = status_find;
+ }
+ if(!has_more_results){
+ state->find_is_done = true;
+ if(state->request_counter == 0){
+ state->handler(state->status); //exit
+ }
+ return false;
+ }
+ return true;
+ };
+
+ //Asynchronous call to Find
+ fs->Find(uri->get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
+
+ /* block until promise is set */
+ hdfs::Status status = future.get();
+ if (!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_stat.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_stat.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_stat.cc
new file mode 100644
index 0000000..c910571
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_stat.cc
@@ -0,0 +1,91 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_rm [OPTION] FILE"
+ << std::endl
+ << std::endl << "Display FILE status."
+ << std::endl
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_rm hdfs://localhost.localdomain:8020/dir/file"
+ << std::endl << "hdfs_rm -R /dir1/dir2"
+ << std::endl;
+}
+
+int main(int argc, char *argv[]) {
+ //We should have at least 2 arguments
+ if (argc < 2) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ int input;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "h")) != -1) {
+ switch (input)
+ {
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ case '?':
+ if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+ std::string uri_path = argv[optind];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ hdfs::StatInfo stat_info;
+ hdfs::Status status = fs->GetFileInfo(uri->get_path(), stat_info);
+ if (!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ std::cout << stat_info.str() << std::endl;
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_tail.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_tail.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_tail.cc
new file mode 100644
index 0000000..2fc1969
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_tail.cc
@@ -0,0 +1,128 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_tail [OPTION] FILE"
+ << std::endl
+ << std::endl << "Displays last kilobyte of the file to stdout."
+ << std::endl
+ << std::endl << " -f output appended data as the file grows, as in Unix"
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_tail hdfs://localhost.localdomain:8020/dir/file"
+ << std::endl << "hdfs_tail /dir/file"
+ << std::endl;
+}
+
+#define TAIL_SIZE 1024
+#define REFRESH_RATE 1 //seconds
+
+int main(int argc, char *argv[]) {
+ if (argc < 2) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ bool follow = false;
+ int input;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "hf")) != -1) {
+ switch (input)
+ {
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ case 'f':
+ follow = true;
+ break;
+ case '?':
+ if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ std::string uri_path = argv[optind];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), false);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ //We need to get the size of the file using stat
+ hdfs::StatInfo stat_info;
+ hdfs::Status status = fs->GetFileInfo(uri->get_path(), stat_info);
+ if (!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ //Determine where to start reading
+ off_t offset = 0;
+ if(stat_info.length > TAIL_SIZE){
+ offset = stat_info.length - TAIL_SIZE;
+ }
+
+ do {
+ off_t current_length = (off_t) stat_info.length;
+ readFile(fs, uri->get_path(), offset, stdout, false);
+
+ //Exit if -f flag was not set
+ if(!follow){
+ break;
+ }
+
+ do{
+ //Sleep for the REFRESH_RATE
+ sleep(REFRESH_RATE);
+ //Use stat to check the new filesize.
+ status = fs->GetFileInfo(uri->get_path(), stat_info);
+ if (!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ //If file became longer, loop back and print the difference
+ }
+ while((off_t) stat_info.length <= current_length);
+ } while (true);
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/52] [abbrv] hadoop git commit: HDFS-10898: libhdfs++: Make log
levels consistent. Contributed by James Clampffer
Posted by jh...@apache.org.
HDFS-10898: libhdfs++: Make log levels consistent. Contributed by James Clampffer
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7ebecaee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7ebecaee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7ebecaee
Branch: refs/heads/HDFS-8707
Commit: 7ebecaeeded9ef6f0d9dff3e8994cf6eaad18920
Parents: 2a42eeb
Author: James <jh...@apache.org>
Authored: Wed Oct 5 13:55:59 2016 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../src/main/native/libhdfspp/lib/fs/filehandle.cc | 10 +++++-----
.../src/main/native/libhdfspp/lib/fs/filesystem.cc | 16 ++++++++--------
2 files changed, 13 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ebecaee/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
index 8f1a82c..03c517b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
@@ -50,7 +50,7 @@ FileHandleImpl::FileHandleImpl(const std::string & cluster_name,
void FileHandleImpl::PositionRead(
void *buf, size_t buf_size, uint64_t offset,
const std::function<void(const Status &, size_t)> &handler) {
- LOG_TRACE(kFileHandle, << "FileHandleImpl::PositionRead("
+ LOG_DEBUG(kFileHandle, << "FileHandleImpl::PositionRead("
<< FMT_THIS_ADDR << ", buf=" << buf
<< ", buf_size=" << buf_size << ") called");
@@ -76,7 +76,7 @@ void FileHandleImpl::PositionRead(
}
Status FileHandleImpl::PositionRead(void *buf, size_t buf_size, off_t offset, size_t *bytes_read) {
- LOG_TRACE(kFileHandle, << "FileHandleImpl::[sync]PositionRead("
+ LOG_DEBUG(kFileHandle, << "FileHandleImpl::[sync]PositionRead("
<< FMT_THIS_ADDR << ", buf=" << buf
<< ", buf_size=" << buf_size
<< ", offset=" << offset << ") called");
@@ -104,7 +104,7 @@ Status FileHandleImpl::PositionRead(void *buf, size_t buf_size, off_t offset, si
}
Status FileHandleImpl::Read(void *buf, size_t buf_size, size_t *bytes_read) {
- LOG_TRACE(kFileHandle, << "FileHandleImpl::Read("
+ LOG_DEBUG(kFileHandle, << "FileHandleImpl::Read("
<< FMT_THIS_ADDR << ", buf=" << buf
<< ", buf_size=" << buf_size << ") called");
@@ -118,7 +118,7 @@ Status FileHandleImpl::Read(void *buf, size_t buf_size, size_t *bytes_read) {
}
Status FileHandleImpl::Seek(off_t *offset, std::ios_base::seekdir whence) {
- LOG_TRACE(kFileHandle, << "FileHandleImpl::Seek("
+ LOG_DEBUG(kFileHandle, << "FileHandleImpl::Seek("
<< ", offset=" << *offset << ", ...) called");
if(cancel_state_->is_canceled()) {
@@ -173,7 +173,7 @@ void FileHandleImpl::AsyncPreadSome(
using ::hadoop::hdfs::DatanodeInfoProto;
using ::hadoop::hdfs::LocatedBlockProto;
- LOG_TRACE(kFileHandle, << "FileHandleImpl::AsyncPreadSome("
+ LOG_DEBUG(kFileHandle, << "FileHandleImpl::AsyncPreadSome("
<< FMT_THIS_ADDR << ", ...) called");
if(cancel_state_->is_canceled()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ebecaee/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
index aa6f4ba..72b30e5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
@@ -281,15 +281,15 @@ int FileSystemImpl::AddWorkerThread() {
void FileSystemImpl::Open(
const std::string &path,
const std::function<void(const Status &, FileHandle *)> &handler) {
- LOG_INFO(kFileSystem, << "FileSystemImpl::Open("
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl::Open("
<< FMT_THIS_ADDR << ", path="
<< path << ") called");
nn_.GetBlockLocations(path, 0, std::numeric_limits<int64_t>::max(), [this, path, handler](const Status &stat, std::shared_ptr<const struct FileInfo> file_info) {
if(!stat.ok()) {
- LOG_INFO(kFileSystem, << "FileSystemImpl::Open failed to get block locations. status=" << stat.ToString());
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl::Open failed to get block locations. status=" << stat.ToString());
if(stat.get_server_exception_type() == Status::kStandbyException) {
- LOG_INFO(kFileSystem, << "Operation not allowed on standby datanode");
+ LOG_DEBUG(kFileSystem, << "Operation not allowed on standby datanode");
}
}
handler(stat, stat.ok() ? new FileHandleImpl(cluster_name_, path, &io_service_->io_service(), client_name_, file_info, bad_node_tracker_, event_handlers_)
@@ -299,7 +299,7 @@ void FileSystemImpl::Open(
Status FileSystemImpl::Open(const std::string &path,
FileHandle **handle) {
- LOG_INFO(kFileSystem, << "FileSystemImpl::[sync]Open("
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl::[sync]Open("
<< FMT_THIS_ADDR << ", path="
<< path << ") called");
@@ -652,7 +652,7 @@ void FileSystemImpl::GetListingShim(const Status &stat, const std::vector<StatIn
void FileSystemImpl::GetListing(
const std::string &path,
const std::function<bool(const Status &, const std::vector<StatInfo> &, bool)> &handler) {
- LOG_INFO(kFileSystem, << "FileSystemImpl::GetListing("
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl::GetListing("
<< FMT_THIS_ADDR << ", path="
<< path << ") called");
@@ -665,7 +665,7 @@ void FileSystemImpl::GetListing(
}
Status FileSystemImpl::GetListing(const std::string &path, std::vector<StatInfo> * stat_infos) {
- LOG_INFO(kFileSystem, << "FileSystemImpl::[sync]GetListing("
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl::[sync]GetListing("
<< FMT_THIS_ADDR << ", path="
<< path << ") called");
@@ -997,7 +997,7 @@ void FileSystemImpl::FindShim(const Status &stat, const std::vector<StatInfo> &
void FileSystemImpl::Find(
const std::string &path, const std::string &name, const uint32_t maxdepth,
const std::function<bool(const Status &, const std::vector<StatInfo> &, bool)> &handler) {
- LOG_INFO(kFileSystem, << "FileSystemImpl::Find("
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl::Find("
<< FMT_THIS_ADDR << ", path="
<< path << ", name="
<< name << ") called");
@@ -1015,7 +1015,7 @@ void FileSystemImpl::Find(
}
Status FileSystemImpl::Find(const std::string &path, const std::string &name, const uint32_t maxdepth, std::vector<StatInfo> * stat_infos) {
- LOG_INFO(kFileSystem, << "FileSystemImpl::[sync]Find("
+ LOG_DEBUG(kFileSystem, << "FileSystemImpl::[sync]Find("
<< FMT_THIS_ADDR << ", path="
<< path << ", name="
<< name << ") called");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/52] [abbrv] hadoop git commit: HDFS-11305: libhdfs++: Log
Datanode information when reading an HDFS block. Contributed by Xiaowei Zhu.
Posted by jh...@apache.org.
HDFS-11305: libhdfs++: Log Datanode information when reading an HDFS block. Contributed by Xiaowei Zhu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/69d55340
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/69d55340
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/69d55340
Branch: refs/heads/HDFS-8707
Commit: 69d55340f89a9bf2efc16b00b24c1ebf303e89fe
Parents: a30cf6a
Author: James <jh...@apache.org>
Authored: Tue Jan 10 11:25:56 2017 -0500
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../src/main/native/libhdfspp/lib/fs/filehandle.cc | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69d55340/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
index 03c517b..1f553a0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
@@ -225,6 +225,12 @@ void FileHandleImpl::AsyncPreadSome(
DatanodeInfoProto &chosen_dn = *it;
+ std::string dnIpAddr = chosen_dn.id().ipaddr();
+ std::string dnHostName = chosen_dn.id().hostname();
+ LOG_DEBUG(kFileHandle, << "FileHandleImpl::AsyncPreadSome("
+ << FMT_THIS_ADDR << "), ...) Datanode hostname=" << dnHostName << ", IP Address=" << dnIpAddr
+ << ", file path=\"" << path_ << "\", offset=" << offset);
+
uint64_t offset_within_block = offset - block->offset();
uint64_t size_within_block = std::min<uint64_t>(
block->b().numbytes() - offset_within_block, asio::buffer_size(buffers));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/52] [abbrv] hadoop git commit: HDFS-12237: libhdfs++:
PROTOC_IS_COMPATIBLE check fails if protobuf library is built from source.
Contributed by Anatoli Shein.
Posted by jh...@apache.org.
HDFS-12237: libhdfs++: PROTOC_IS_COMPATIBLE check fails if protobuf library is built from source. Contributed by Anatoli Shein.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eeb49d0c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eeb49d0c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eeb49d0c
Branch: refs/heads/HDFS-8707
Commit: eeb49d0ca7149b4dd4204f952983cdc6f0471f00
Parents: 22ea06a
Author: James Clampffer <ja...@hp.com>
Authored: Mon Aug 14 15:24:13 2017 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../src/main/native/libhdfspp/CMakeLists.txt | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eeb49d0c/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
index 812af4b..2bcb49a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
@@ -65,6 +65,7 @@ endif (NOT THREAD_LOCAL_SUPPORTED)
# Check if PROTOC library was compiled with the compatible compiler by trying
# to compile some dummy code
unset (PROTOC_IS_COMPATIBLE CACHE)
+set (CMAKE_REQUIRED_INCLUDES ${PROTOBUF_INCLUDE_DIRS})
set (CMAKE_REQUIRED_LIBRARIES ${PROTOBUF_LIBRARY} ${PROTOBUF_PROTOC_LIBRARY})
check_cxx_source_compiles(
"#include <google/protobuf/io/printer.h>
@@ -77,8 +78,8 @@ check_cxx_source_compiles(
}"
PROTOC_IS_COMPATIBLE)
if (NOT PROTOC_IS_COMPATIBLE)
- message(FATAL_ERROR
- "FATAL ERROR: the Protocol Buffers Library and the Libhdfs++ Library must both be compiled \
+ message(WARNING
+ "WARNING: the Protocol Buffers Library and the Libhdfs++ Library must both be compiled \
with the same (or compatible) compiler. Normally only the same major versions of the same \
compiler are compatible with each other.")
endif (NOT PROTOC_IS_COMPATIBLE)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/52] [abbrv] hadoop git commit: HDFS-10754: libhdfs++: Create
tools directory and implement hdfs_cat, hdfs_chgrp, hdfs_chown,
hdfs_chmod and hdfs_find. Contributed by Anatoli Shein.
Posted by jh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c
index fd82da3..ddba67f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c
@@ -492,6 +492,10 @@ int hdfsFreeBlockLocations(struct hdfsBlockLocations * locations) {
return libhdfspp_hdfsFreeBlockLocations(locations);
}
+hdfsFileInfo *hdfsFind(hdfsFS fs, const char* path, const char* name, uint32_t *numEntries) {
+ return (hdfsFileInfo *)libhdfspp_hdfsFind(fs->libhdfsppRep, path, name, numEntries);
+}
+
int hdfsCreateSnapshot(hdfsFS fs, const char* path, const char* name) {
return libhdfspp_hdfsCreateSnapshot(fs->libhdfsppRep, path, name);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h
index 481ed68..644ff13 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h
@@ -94,6 +94,7 @@
#undef hdfsCancel
#undef hdfsGetBlockLocations
#undef hdfsFreeBlockLocations
+#undef hdfsFind
#undef hdfsCreateSnapshot
#undef hdfsDeleteSnapshot
#undef hdfsAllowSnapshot
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h
index a1e4483..c186d63 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h
@@ -94,6 +94,7 @@
#define hdfsCancel libhdfspp_hdfsCancel
#define hdfsGetBlockLocations libhdfspp_hdfsGetBlockLocations
#define hdfsFreeBlockLocations libhdfspp_hdfsFreeBlockLocations
+#define hdfsFind libhdfspp_hdfsFind
#define hdfsCreateSnapshot libhdfspp_hdfsCreateSnapshot
#define hdfsDeleteSnapshot libhdfspp_hdfsDeleteSnapshot
#define hdfsAllowSnapshot libhdfspp_hdfsAllowSnapshot
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt
new file mode 100644
index 0000000..f0817eb
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt
@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Default LIBHDFSPP_DIR to the default install location. You can override
+# it by add -DLIBHDFSPP_DIR=... to your cmake invocation
+set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX})
+
+include_directories( ${LIBHDFSPP_DIR}/include )
+link_directories( ${LIBHDFSPP_DIR}/lib )
+
+add_library(tools_common_obj OBJECT tools_common.cpp)
+add_library(tools_common $<TARGET_OBJECTS:tools_common_obj>)
+
+add_executable(hdfs_cat hdfs_cat.cpp)
+target_link_libraries(hdfs_cat tools_common hdfspp)
+
+add_executable(hdfs_chgrp hdfs_chgrp.cpp)
+target_link_libraries(hdfs_chgrp tools_common hdfspp)
+
+add_executable(hdfs_chown hdfs_chown.cpp)
+target_link_libraries(hdfs_chown tools_common hdfspp)
+
+add_executable(hdfs_chmod hdfs_chmod.cpp)
+target_link_libraries(hdfs_chmod tools_common hdfspp)
+
+add_executable(hdfs_find hdfs_find.cpp)
+target_link_libraries(hdfs_find tools_common hdfspp)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_cat.cpp
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_cat.cpp b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_cat.cpp
new file mode 100644
index 0000000..166a7bf
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_cat.cpp
@@ -0,0 +1,120 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_cat [OPTION] FILE"
+ << std::endl
+ << std::endl << "Concatenate FILE to standard output."
+ << std::endl
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_cat hdfs://localhost.localdomain:9433/dir/file"
+ << std::endl << "hdfs_cat /dir/file"
+ << std::endl;
+}
+
+#define BUF_SIZE 4096
+
+int main(int argc, char *argv[]) {
+ if (argc != 2) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ int input;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "h")) != -1) {
+ switch (input)
+ {
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ break;
+ case '?':
+ if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ std::string uri_path = argv[optind];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ //TODO: HDFS-9539 Currently options can be returned empty
+ hdfs::Options options = *hdfs::getOptions();
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), options);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ hdfs::FileHandle *file_raw = nullptr;
+ hdfs::Status status = fs->Open(uri->get_path(), &file_raw);
+ if (!status.ok()) {
+ std::cerr << "Could not open file " << uri->get_path() << ". " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ //wrapping file_raw into a unique pointer to guarantee deletion
+ std::unique_ptr<hdfs::FileHandle> file(file_raw);
+
+ char input_buffer[BUF_SIZE];
+ ssize_t total_bytes_read = 0;
+ size_t last_bytes_read = 0;
+
+ do{
+ //Reading file chunks
+ status = file->Read(input_buffer, sizeof(input_buffer), &last_bytes_read);
+ if(status.ok()) {
+ //Writing file chunks to stdout
+ fwrite(input_buffer, last_bytes_read, 1, stdout);
+ total_bytes_read += last_bytes_read;
+ } else {
+ if(status.is_invalid_offset()){
+ //Reached the end of the file
+ break;
+ } else {
+ std::cerr << "Error reading the file: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ }
+ } while (last_bytes_read > 0);
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cpp
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cpp b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cpp
new file mode 100644
index 0000000..2bb6843
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cpp
@@ -0,0 +1,196 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include <future>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_chgrp [OPTION] GROUP FILE"
+ << std::endl
+ << std::endl << "Change the group association of each FILE to GROUP."
+ << std::endl << "The user must be the owner of files. Additional information is in the Permissions Guide:"
+ << std::endl << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html"
+ << std::endl
+ << std::endl << " -R operate on files and directories recursively"
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_chgrp -R new_group hdfs://localhost.localdomain:9433/dir/file"
+ << std::endl << "hdfs_chgrp new_group /dir/file"
+ << std::endl;
+}
+
+struct SetOwnerState {
+ const std::string username;
+ const std::string groupname;
+ const std::function<void(const hdfs::Status &)> handler;
+ //The request counter is incremented once every time SetOwner async call is made
+ uint64_t request_counter;
+ //This boolean will be set when find returns the last result
+ bool find_is_done;
+ //Final status to be returned
+ hdfs::Status status;
+ //Shared variables will need protection with a lock
+ std::mutex lock;
+ SetOwnerState(const std::string & username_, const std::string & groupname_,
+ const std::function<void(const hdfs::Status &)> & handler_,
+ uint64_t request_counter_, bool find_is_done_)
+ : username(username_),
+ groupname(groupname_),
+ handler(handler_),
+ request_counter(request_counter_),
+ find_is_done(find_is_done_),
+ status(),
+ lock() {
+ }
+};
+
+int main(int argc, char *argv[]) {
+ //We should have 3 or 4 parameters
+ if (argc != 3 && argc != 4) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ bool recursive = false;
+ int input;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "Rh")) != -1) {
+ switch (input)
+ {
+ case 'R':
+ recursive = 1;
+ break;
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ break;
+ case '?':
+ if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+ std::string group = argv[optind];
+ //Owner stays the same, just group association changes.
+ std::string owner = "";
+ std::string uri_path = argv[optind + 1];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ //TODO: HDFS-9539 Currently options can be returned empty
+ hdfs::Options options = *hdfs::getOptions();
+
+ //TODO: HDFS-9539 - until then we increase the time-out to allow all recursive async calls to finish
+ options.rpc_timeout = std::numeric_limits<int>::max();
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), options);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ /* wrap async FileSystem::SetOwner with promise to make it a blocking call */
+ std::shared_ptr<std::promise<hdfs::Status>> promise = std::make_shared<std::promise<hdfs::Status>>();
+ std::future<hdfs::Status> future(promise->get_future());
+ auto handler = [promise](const hdfs::Status &s) {
+ promise->set_value(s);
+ };
+
+ if(!recursive){
+ fs->SetOwner(uri->get_path(), owner, group, handler);
+ }
+ else {
+ //Allocating shared state, which includes:
+ //username and groupname to be set, handler to be called, request counter, and a boolean to keep track if find is done
+ std::shared_ptr<SetOwnerState> state = std::make_shared<SetOwnerState>(owner, group, handler, 0, false);
+
+ // Keep requesting more from Find until we process the entire listing. Call handler when Find is done and reques counter is 0.
+ // Find guarantees that the handler will only be called once at a time so we do not need locking in handlerFind.
+ auto handlerFind = [fs, state](const hdfs::Status &status_find, const std::vector<hdfs::StatInfo> & stat_infos, bool has_more_results) -> bool {
+
+ //For each result returned by Find we call async SetOwner with the handler below.
+ //SetOwner DOES NOT guarantee that the handler will only be called once at a time, so we DO need locking in handlerSetOwner.
+ auto handlerSetOwner = [state](const hdfs::Status &status_set_owner) {
+ std::lock_guard<std::mutex> guard(state->lock);
+
+ //Decrement the counter once since we are done with this async call
+ if (!status_set_owner.ok() && state->status.ok()){
+ //We make sure we set state->status only on the first error.
+ state->status = status_set_owner;
+ }
+ state->request_counter--;
+ if(state->request_counter == 0 && state->find_is_done){
+ state->handler(state->status); //exit
+ }
+ };
+ if(!stat_infos.empty() && state->status.ok()) {
+ for (hdfs::StatInfo const& s : stat_infos) {
+ //Launch an asynchronous call to SetOwner for every returned result
+ state->request_counter++;
+ fs->SetOwner(s.full_path, state->username, state->groupname, handlerSetOwner);
+ }
+ }
+
+ //Lock this section because handlerSetOwner might be accessing the same
+ //shared variables simultaneously
+ std::lock_guard<std::mutex> guard(state->lock);
+ if (!status_find.ok() && state->status.ok()){
+ //We make sure we set state->status only on the first error.
+ state->status = status_find;
+ }
+ if(!has_more_results){
+ state->find_is_done = true;
+ if(state->request_counter == 0){
+ state->handler(state->status); //exit
+ }
+ return false;
+ }
+ return true;
+ };
+
+ //Asynchronous call to Find
+ fs->Find(uri->get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
+ }
+
+ /* block until promise is set */
+ hdfs::Status status = future.get();
+ if (!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cpp
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cpp b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cpp
new file mode 100644
index 0000000..0a001d6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cpp
@@ -0,0 +1,194 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include <future>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_chmod [OPTION] <MODE[,MODE]... | OCTALMODE> FILE"
+ << std::endl
+ << std::endl << "Change the permissions of each FILE to MODE."
+ << std::endl << "The user must be the owner of the file, or else a super-user."
+ << std::endl << "Additional information is in the Permissions Guide:"
+ << std::endl << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html"
+ << std::endl
+ << std::endl << " -R operate on files and directories recursively"
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_chmod -R 755 hdfs://localhost.localdomain:9433/dir/file"
+ << std::endl << "hdfs_chmod 777 /dir/file"
+ << std::endl;
+}
+
+struct SetPermissionState {
+ const uint16_t permissions;
+ const std::function<void(const hdfs::Status &)> handler;
+ //The request counter is incremented once every time SetOwner async call is made
+ uint64_t request_counter;
+ //This boolean will be set when find returns the last result
+ bool find_is_done;
+ //Final status to be returned
+ hdfs::Status status;
+ //Shared variables will need protection with a lock
+ std::mutex lock;
+ SetPermissionState(const uint16_t permissions_, const std::function<void(const hdfs::Status &)> & handler_,
+ uint64_t request_counter_, bool find_is_done_)
+ : permissions(permissions_),
+ handler(handler_),
+ request_counter(request_counter_),
+ find_is_done(find_is_done_),
+ status(),
+ lock() {
+ }
+};
+
+int main(int argc, char *argv[]) {
+ //We should have 3 or 4 parameters
+ if (argc != 3 && argc != 4) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ bool recursive = false;
+ int input;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "Rh")) != -1) {
+ switch (input)
+ {
+ case 'R':
+ recursive = 1;
+ break;
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ break;
+ case '?':
+ if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+ std::string permissions = argv[optind];
+ std::string uri_path = argv[optind + 1];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ //TODO: HDFS-9539 Currently options can be returned empty
+ hdfs::Options options = *hdfs::getOptions();
+
+ //TODO: HDFS-9539 - until then we increase the time-out to allow all recursive async calls to finish
+ options.rpc_timeout = std::numeric_limits<int>::max();
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), options);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ /* wrap async FileSystem::SetPermission with promise to make it a blocking call */
+ std::shared_ptr<std::promise<hdfs::Status>> promise = std::make_shared<std::promise<hdfs::Status>>();
+ std::future<hdfs::Status> future(promise->get_future());
+ auto handler = [promise](const hdfs::Status &s) {
+ promise->set_value(s);
+ };
+
+ //strtol() is reading the value with base 8, NULL because we are reading in just one value.
+ uint16_t perm = strtol(permissions.c_str(), NULL, 8);
+ if(!recursive){
+ fs->SetPermission(uri->get_path(), perm, handler);
+ }
+ else {
+ //Allocating shared state, which includes:
+ //username and groupname to be set, handler to be called, request counter, and a boolean to keep track if find is done
+ std::shared_ptr<SetPermissionState> state = std::make_shared<SetPermissionState>(perm, handler, 0, false);
+
+ // Keep requesting more from Find until we process the entire listing. Call handler when Find is done and reques counter is 0.
+ // Find guarantees that the handler will only be called once at a time so we do not need locking in handlerFind.
+ auto handlerFind = [fs, state](const hdfs::Status &status_find, const std::vector<hdfs::StatInfo> & stat_infos, bool has_more_results) -> bool {
+
+ //For each result returned by Find we call async SetOwner with the handler below.
+ //SetOwner DOES NOT guarantee that the handler will only be called once at a time, so we DO need locking in handlerSetOwner.
+ auto handlerSetOwner = [state](const hdfs::Status &status_set_owner) {
+ std::lock_guard<std::mutex> guard(state->lock);
+
+ //Decrement the counter once since we are done with this async call
+ if (!status_set_owner.ok() && state->status.ok()){
+ //We make sure we set state->status only on the first error.
+ state->status = status_set_owner;
+ }
+ state->request_counter--;
+ if(state->request_counter == 0 && state->find_is_done){
+ state->handler(state->status); //exit
+ }
+ };
+ if(!stat_infos.empty() && state->status.ok()) {
+ for (hdfs::StatInfo const& s : stat_infos) {
+ //Launch an asynchronous call to SetOwner for every returned result
+ state->request_counter++;
+ fs->SetPermission(s.full_path, state->permissions, handlerSetOwner);
+ }
+ }
+
+ //Lock this section because handlerSetOwner might be accessing the same
+ //shared variables simultaneously
+ std::lock_guard<std::mutex> guard(state->lock);
+ if (!status_find.ok() && state->status.ok()){
+ //We make sure we set state->status only on the first error.
+ state->status = status_find;
+ }
+ if(!has_more_results){
+ state->find_is_done = true;
+ if(state->request_counter == 0){
+ state->handler(state->status); //exit
+ }
+ return false;
+ }
+ return true;
+ };
+
+ //Asynchronous call to Find
+ fs->Find(uri->get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
+ }
+
+ /* block until promise is set */
+ hdfs::Status status = future.get();
+ if (!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cpp
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cpp b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cpp
new file mode 100644
index 0000000..08724c6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cpp
@@ -0,0 +1,206 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include <future>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_chown [OPTION] [OWNER][:[GROUP]] FILE"
+ << std::endl
+ << std::endl << "Change the owner and/or group of each FILE to OWNER and/or GROUP."
+ << std::endl << "The user must be a super-user. Additional information is in the Permissions Guide:"
+ << std::endl << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html"
+ << std::endl
+ << std::endl << " -R operate on files and directories recursively"
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Owner is unchanged if missing. Group is unchanged if missing."
+ << std::endl << "OWNER and GROUP may be numeric as well as symbolic."
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_chown -R new_owner:new_group hdfs://localhost.localdomain:9433/dir/file"
+ << std::endl << "hdfs_chown new_owner /dir/file"
+ << std::endl;
+}
+
+struct SetOwnerState {
+ const std::string username;
+ const std::string groupname;
+ const std::function<void(const hdfs::Status &)> handler;
+ //The request counter is incremented once every time SetOwner async call is made
+ uint64_t request_counter;
+ //This boolean will be set when find returns the last result
+ bool find_is_done;
+ //Final status to be returned
+ hdfs::Status status;
+ //Shared variables will need protection with a lock
+ std::mutex lock;
+ SetOwnerState(const std::string & username_, const std::string & groupname_,
+ const std::function<void(const hdfs::Status &)> & handler_,
+ uint64_t request_counter_, bool find_is_done_)
+ : username(username_),
+ groupname(groupname_),
+ handler(handler_),
+ request_counter(request_counter_),
+ find_is_done(find_is_done_),
+ status(),
+ lock() {
+ }
+};
+
+int main(int argc, char *argv[]) {
+ //We should have 3 or 4 parameters
+ if (argc != 3 && argc != 4) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ bool recursive = false;
+ int input;
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "Rh")) != -1) {
+ switch (input)
+ {
+ case 'R':
+ recursive = 1;
+ break;
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ break;
+ case '?':
+ if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+ std::string owner_and_group = argv[optind];
+ std::string uri_path = argv[optind + 1];
+
+ std::string owner, group;
+ size_t owner_end = owner_and_group.find(":");
+ if(owner_end == std::string::npos) {
+ owner = owner_and_group;
+ } else {
+ owner = owner_and_group.substr(0, owner_end);
+ group = owner_and_group.substr(owner_end + 1);
+ }
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ //TODO: HDFS-9539 Currently options can be returned empty
+ hdfs::Options options = *hdfs::getOptions();
+
+ //TODO: HDFS-9539 - until then we increase the time-out to allow all recursive async calls to finish
+ options.rpc_timeout = std::numeric_limits<int>::max();
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), options);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ /* wrap async FileSystem::SetOwner with promise to make it a blocking call */
+ std::shared_ptr<std::promise<hdfs::Status>> promise = std::make_shared<std::promise<hdfs::Status>>();
+ std::future<hdfs::Status> future(promise->get_future());
+ auto handler = [promise](const hdfs::Status &s) {
+ promise->set_value(s);
+ };
+
+ if(!recursive){
+ fs->SetOwner(uri->get_path(), owner, group, handler);
+ }
+ else {
+ //Allocating shared state, which includes:
+ //username and groupname to be set, handler to be called, request counter, and a boolean to keep track if find is done
+ std::shared_ptr<SetOwnerState> state = std::make_shared<SetOwnerState>(owner, group, handler, 0, false);
+
+ // Keep requesting more from Find until we process the entire listing. Call handler when Find is done and reques counter is 0.
+ // Find guarantees that the handler will only be called once at a time so we do not need locking in handlerFind.
+ auto handlerFind = [fs, state](const hdfs::Status &status_find, const std::vector<hdfs::StatInfo> & stat_infos, bool has_more_results) -> bool {
+
+ //For each result returned by Find we call async SetOwner with the handler below.
+ //SetOwner DOES NOT guarantee that the handler will only be called once at a time, so we DO need locking in handlerSetOwner.
+ auto handlerSetOwner = [state](const hdfs::Status &status_set_owner) {
+ std::lock_guard<std::mutex> guard(state->lock);
+
+ //Decrement the counter once since we are done with this async call
+ if (!status_set_owner.ok() && state->status.ok()){
+ //We make sure we set state->status only on the first error.
+ state->status = status_set_owner;
+ }
+ state->request_counter--;
+ if(state->request_counter == 0 && state->find_is_done){
+ state->handler(state->status); //exit
+ }
+ };
+ if(!stat_infos.empty() && state->status.ok()) {
+ for (hdfs::StatInfo const& s : stat_infos) {
+ //Launch an asynchronous call to SetOwner for every returned result
+ state->request_counter++;
+ fs->SetOwner(s.full_path, state->username, state->groupname, handlerSetOwner);
+ }
+ }
+
+ //Lock this section because handlerSetOwner might be accessing the same
+ //shared variables simultaneously
+ std::lock_guard<std::mutex> guard(state->lock);
+ if (!status_find.ok() && state->status.ok()){
+ //We make sure we set state->status only on the first error.
+ state->status = status_find;
+ }
+ if(!has_more_results){
+ state->find_is_done = true;
+ if(state->request_counter == 0){
+ state->handler(state->status); //exit
+ }
+ return false;
+ }
+ return true;
+ };
+
+ //Asynchronous call to Find
+ fs->Find(uri->get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
+ }
+
+ /* block until promise is set */
+ hdfs::Status status = future.get();
+ if (!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_find.cpp
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_find.cpp b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_find.cpp
new file mode 100644
index 0000000..eca79c6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_find.cpp
@@ -0,0 +1,156 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include <google/protobuf/stubs/common.h>
+#include <unistd.h>
+#include <future>
+#include "tools_common.h"
+
+void usage(){
+ std::cout << "Usage: hdfs_find [OPTION] PATH"
+ << std::endl
+ << std::endl << "Finds all files recursively starting from the"
+ << std::endl << "specified PATH and prints their file paths."
+ << std::endl << "This hdfs_find tool mimics the POSIX find."
+ << std::endl
+ << std::endl << "Both PATH and NAME can have wild-cards."
+ << std::endl
+ << std::endl << " -n NAME if provided all results will be matching the NAME pattern"
+ << std::endl << " otherwise, the implicit '*' will be used"
+ << std::endl << " NAME allows wild-cards"
+ << std::endl
+ << std::endl << " -m MAX_DEPTH if provided the maximum depth to recurse after the end of"
+ << std::endl << " the path is reached will be limited by MAX_DEPTH"
+ << std::endl << " otherwise, the maximum depth to recurse is unbound"
+ << std::endl << " MAX_DEPTH can be set to 0 for pure globbing and ignoring"
+ << std::endl << " the NAME option (no recursion after the end of the path)"
+ << std::endl
+ << std::endl << " -h display this help and exit"
+ << std::endl
+ << std::endl << "Examples:"
+ << std::endl << "hdfs_find hdfs://localhost.localdomain:9433/dir?/tree* -n some?file*name"
+ << std::endl << "hdfs_find / -n file_name -m 3"
+ << std::endl;
+}
+
+int main(int argc, char *argv[]) {
+ //We should have at least 2 arguments
+ if (argc < 2) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ int input;
+ //If NAME is not specified we use implicit "*"
+ std::string name = "*";
+ //If MAX_DEPTH is not specified we use the max value of uint_32_t
+ uint32_t max_depth = hdfs::FileSystem::GetDefaultFindMaxDepth();
+
+ //Using GetOpt to read in the values
+ opterr = 0;
+ while ((input = getopt(argc, argv, "hn:m:")) != -1) {
+ switch (input)
+ {
+ case 'h':
+ usage();
+ exit(EXIT_SUCCESS);
+ break;
+ case 'n':
+ name = optarg;
+ break;
+ case 'm':
+ max_depth = std::stoi(optarg);
+ break;
+ case '?':
+ if (optopt == 'n' || optopt == 'm')
+ std::cerr << "Option -" << (char) optopt << " requires an argument." << std::endl;
+ else if (isprint(optopt))
+ std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
+ else
+ std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
+ usage();
+ exit(EXIT_FAILURE);
+ default:
+ exit(EXIT_FAILURE);
+ }
+ }
+ std::string uri_path = argv[optind];
+
+ //Building a URI object from the given uri_path
+ hdfs::optional<hdfs::URI> uri = hdfs::URI::parse_from_string(uri_path);
+ if (!uri) {
+ std::cerr << "Malformed URI: " << uri_path << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ //TODO: HDFS-9539 Currently options can be returned empty
+ hdfs::Options options = *hdfs::getOptions();
+
+ //TODO: HDFS-9539 - until then we increase the time-out to allow all recursive async calls to finish
+ options.rpc_timeout = std::numeric_limits<int>::max();
+
+ std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri.value(), options);
+ if (!fs) {
+ std::cerr << "Could not connect the file system. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::promise<void> promise;
+ std::future<void> future(promise.get_future());
+ hdfs::Status status = hdfs::Status::OK();
+
+ /**
+ * Keep requesting more until we get the entire listing. Set the promise
+ * when we have the entire listing to stop.
+ *
+ * Find guarantees that the handler will only be called once at a time,
+ * so we do not need any locking here
+ */
+ auto handler = [&promise, &status]
+ (const hdfs::Status &s, const std::vector<hdfs::StatInfo> & si, bool has_more_results) -> bool {
+ //Print result chunks as they arrive
+ if(!si.empty()) {
+ for (hdfs::StatInfo const& s : si) {
+ std::cout << s.full_path << std::endl;
+ }
+ }
+ if(!s.ok() && status.ok()){
+ //We make sure we set 'status' only on the first error.
+ status = s;
+ }
+ if (!has_more_results) {
+ promise.set_value(); //set promise
+ return false; //request stop sending results
+ }
+ return true; //request more results
+ };
+
+ //Asynchronous call to Find
+ fs->Find(uri->get_path(), name, max_depth, handler);
+
+ //block until promise is set
+ future.get();
+ if(!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ }
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cpp
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cpp b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cpp
new file mode 100644
index 0000000..af882ce
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cpp
@@ -0,0 +1,70 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#include "tools_common.h"
+
+namespace hdfs {
+
+ std::shared_ptr<hdfs::Options> getOptions() {
+ std::shared_ptr<hdfs::Options> options = std::make_shared<hdfs::Options>();
+ //Setting the config path to the default: "$HADOOP_CONF_DIR" or "/etc/hadoop/conf"
+ hdfs::ConfigurationLoader loader;
+ //Loading default config files core-site.xml and hdfs-site.xml from the config path
+ hdfs::optional<HdfsConfiguration> config = loader.LoadDefaultResources<HdfsConfiguration>();
+ //TODO: HDFS-9539 - after this is resolved, valid config will always be returned.
+ if(config){
+ //Loading options from the config
+ *options = config->GetOptions();
+ }
+ return options;
+ }
+
+ std::shared_ptr<hdfs::FileSystem> doConnect(hdfs::URI & uri, hdfs::Options & options) {
+ IoService * io_service = IoService::New();
+ //Wrapping fs into a shared pointer to guarantee deletion
+ std::shared_ptr<hdfs::FileSystem> fs(hdfs::FileSystem::New(io_service, "", options));
+ if (!fs) {
+ std::cerr << "Could not create FileSystem object. " << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ Status status;
+ //Check if the user supplied the host
+ if(!uri.get_host().empty()){
+ //If port is supplied we use it, otherwise we use the empty string so that it will be looked up in configs.
+ std::string port = (uri.get_port()) ? std::to_string(uri.get_port().value()) : "";
+ status = fs->Connect(uri.get_host(), port);
+ if (!status.ok()) {
+ std::cerr << "Could not connect to " << uri.get_host() << ":" << port << ". " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ } else {
+ status = fs->ConnectToDefaultFs();
+ if (!status.ok()) {
+ if(!options.defaultFS.get_host().empty()){
+ std::cerr << "Error connecting to " << options.defaultFS << ". " << status.ToString() << std::endl;
+ } else {
+ std::cerr << "Error connecting to the cluster: defaultFS is empty. " << status.ToString() << std::endl;
+ }
+ exit(EXIT_FAILURE);
+ }
+ }
+ return fs;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.h
new file mode 100644
index 0000000..858fc4b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.h
@@ -0,0 +1,39 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+#ifndef TOOLS_COMMON_H_
+#define TOOLS_COMMON_H_
+
+#include "hdfspp/hdfspp.h"
+#include "common/hdfs_configuration.h"
+#include "common/configuration_loader.h"
+
+#include <mutex>
+
+namespace hdfs {
+
+ //Pull configurations and get the Options object
+ std::shared_ptr<hdfs::Options> getOptions();
+
+ //Build all necessary objects and perform the connection
+ std::shared_ptr<hdfs::FileSystem> doConnect(hdfs::URI & uri, hdfs::Options & options);
+
+}
+
+#endif
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/52] [abbrv] hadoop git commit: HDFS-10785: libhdfs++: Implement
the rest of the tools. Contributed by Anatoli Schein
Posted by jh...@apache.org.
HDFS-10785: libhdfs++: Implement the rest of the tools. Contributed by Anatoli Schein
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/869317be
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/869317be
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/869317be
Branch: refs/heads/HDFS-8707
Commit: 869317be0a1fdff23be5fc500dcd9ae4ecd7bc29
Parents: 52445fd
Author: James Clampffer <ja...@hp.com>
Authored: Tue Jun 13 11:27:55 2017 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../native/libhdfspp/examples/CMakeLists.txt | 2 +-
.../main/native/libhdfspp/examples/c/cat/cat.c | 4 +-
.../native/libhdfspp/examples/cc/CMakeLists.txt | 22 ++
.../libhdfspp/examples/cc/cat/CMakeLists.txt | 35 ++++
.../native/libhdfspp/examples/cc/cat/cat.cc | 109 ++++++++++
.../examples/cc/connect_cancel/CMakeLists.txt | 27 +++
.../cc/connect_cancel/connect_cancel.cc | 154 ++++++++++++++
.../libhdfspp/examples/cc/find/CMakeLists.txt | 35 ++++
.../native/libhdfspp/examples/cc/find/find.cc | 162 +++++++++++++++
.../examples/cc/gendirs/CMakeLists.txt | 35 ++++
.../libhdfspp/examples/cc/gendirs/gendirs.cc | 144 +++++++++++++
.../libhdfspp/examples/cpp/CMakeLists.txt | 22 --
.../libhdfspp/examples/cpp/cat/CMakeLists.txt | 35 ----
.../native/libhdfspp/examples/cpp/cat/cat.cpp | 109 ----------
.../examples/cpp/connect_cancel/CMakeLists.txt | 29 ---
.../cpp/connect_cancel/connect_cancel.cc | 158 --------------
.../libhdfspp/examples/cpp/find/CMakeLists.txt | 35 ----
.../native/libhdfspp/examples/cpp/find/find.cpp | 162 ---------------
.../examples/cpp/gendirs/CMakeLists.txt | 35 ----
.../libhdfspp/examples/cpp/gendirs/gendirs.cpp | 144 -------------
.../libhdfspp/include/hdfspp/content_summary.h | 48 +++++
.../native/libhdfspp/include/hdfspp/fsinfo.h | 34 ++-
.../native/libhdfspp/include/hdfspp/hdfs_ext.h | 11 +
.../native/libhdfspp/include/hdfspp/hdfspp.h | 25 ++-
.../native/libhdfspp/include/hdfspp/statinfo.h | 46 ++---
.../native/libhdfspp/lib/bindings/c/hdfs.cc | 31 +++
.../native/libhdfspp/lib/common/CMakeLists.txt | 2 +-
.../libhdfspp/lib/common/content_summary.cc | 55 +++++
.../main/native/libhdfspp/lib/common/fsinfo.cc | 61 ++++++
.../native/libhdfspp/lib/common/statinfo.cc | 74 +++++++
.../main/native/libhdfspp/lib/fs/filesystem.cc | 46 ++++-
.../main/native/libhdfspp/lib/fs/filesystem.h | 18 ++
.../native/libhdfspp/lib/fs/filesystem_sync.cc | 52 +++++
.../libhdfspp/lib/fs/namenode_operations.cc | 90 +++++++-
.../libhdfspp/lib/fs/namenode_operations.h | 10 +-
.../native/libhdfspp/tests/hdfs_ext_test.cc | 11 +-
.../src/main/native/libhdfspp/tests/hdfs_shim.c | 4 +
.../libhdfspp/tests/libhdfs_wrapper_undefs.h | 1 +
.../libhdfspp/tests/libhdfspp_wrapper_defines.h | 1 +
.../main/native/libhdfspp/tools/CMakeLists.txt | 65 +++++-
.../libhdfspp/tools/hdfs_allowSnapshot.cc | 90 ++++++++
.../src/main/native/libhdfspp/tools/hdfs_cat.cc | 87 ++++++++
.../main/native/libhdfspp/tools/hdfs_cat.cpp | 120 -----------
.../main/native/libhdfspp/tools/hdfs_chgrp.cc | 189 +++++++++++++++++
.../main/native/libhdfspp/tools/hdfs_chgrp.cpp | 196 ------------------
.../main/native/libhdfspp/tools/hdfs_chmod.cc | 187 +++++++++++++++++
.../main/native/libhdfspp/tools/hdfs_chmod.cpp | 194 -----------------
.../main/native/libhdfspp/tools/hdfs_chown.cc | 199 ++++++++++++++++++
.../main/native/libhdfspp/tools/hdfs_chown.cpp | 206 -------------------
.../native/libhdfspp/tools/hdfs_copyToLocal.cc | 92 +++++++++
.../main/native/libhdfspp/tools/hdfs_count.cc | 97 +++++++++
.../libhdfspp/tools/hdfs_createSnapshot.cc | 99 +++++++++
.../libhdfspp/tools/hdfs_deleteSnapshot.cc | 91 ++++++++
.../src/main/native/libhdfspp/tools/hdfs_df.cc | 93 +++++++++
.../libhdfspp/tools/hdfs_disallowSnapshot.cc | 90 ++++++++
.../src/main/native/libhdfspp/tools/hdfs_du.cc | 180 ++++++++++++++++
.../main/native/libhdfspp/tools/hdfs_find.cc | 150 ++++++++++++++
.../main/native/libhdfspp/tools/hdfs_find.cpp | 156 --------------
.../src/main/native/libhdfspp/tools/hdfs_get.cc | 92 +++++++++
.../src/main/native/libhdfspp/tools/hdfs_ls.cc | 134 ++++++++++++
.../main/native/libhdfspp/tools/hdfs_mkdir.cc | 102 +++++++++
.../native/libhdfspp/tools/hdfs_moveToLocal.cc | 94 +++++++++
.../libhdfspp/tools/hdfs_renameSnapshot.cc | 92 +++++++++
.../src/main/native/libhdfspp/tools/hdfs_rm.cc | 94 +++++++++
.../main/native/libhdfspp/tools/hdfs_setrep.cc | 176 ++++++++++++++++
.../main/native/libhdfspp/tools/hdfs_stat.cc | 91 ++++++++
.../main/native/libhdfspp/tools/hdfs_tail.cc | 128 ++++++++++++
.../main/native/libhdfspp/tools/tools_common.cc | 115 +++++++++++
.../native/libhdfspp/tools/tools_common.cpp | 70 -------
.../main/native/libhdfspp/tools/tools_common.h | 8 +-
70 files changed, 4116 insertions(+), 1739 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/CMakeLists.txt
index 6b680cc..ca3a2c7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/CMakeLists.txt
@@ -17,4 +17,4 @@
#
add_subdirectory(c)
-add_subdirectory(cpp)
+add_subdirectory(cc)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/cat.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/cat.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/cat.c
index 586e0a9..bee5382 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/cat.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/cat.c
@@ -30,6 +30,8 @@
#include "common/util_c.h"
#define SCHEME "hdfs"
+#define BUF_SIZE 1048576 //1 MB
+static char input_buffer[BUF_SIZE];
int main(int argc, char** argv) {
@@ -84,8 +86,6 @@ int main(int argc, char** argv) {
return 1;
}
- char input_buffer[4096];
-
ssize_t read_bytes_count = 0;
ssize_t last_read_bytes = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/CMakeLists.txt
new file mode 100644
index 0000000..8893f03
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/CMakeLists.txt
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_subdirectory(cat)
+add_subdirectory(gendirs)
+add_subdirectory(find)
+add_subdirectory(connect_cancel)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/CMakeLists.txt
new file mode 100644
index 0000000..9ad919a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/CMakeLists.txt
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Default LIBHDFSPP_DIR to the default install location. You can override
+# it by add -DLIBHDFSPP_DIR=... to your cmake invocation
+set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX})
+
+include_directories( ${LIBHDFSPP_DIR}/include )
+link_directories( ${LIBHDFSPP_DIR}/lib )
+
+add_executable(cat_cc cat.cc)
+target_link_libraries(cat_cc hdfspp)
+
+# Several examples in different languages need to produce executables with
+# same names. To allow executables with same names we keep their CMake
+# names different, but specify their executable names as follows:
+set_target_properties( cat_cc
+ PROPERTIES
+ OUTPUT_NAME "cat"
+)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/cat.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/cat.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/cat.cc
new file mode 100644
index 0000000..bca2a94
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/cat.cc
@@ -0,0 +1,109 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+/**
+ * Unix-like cat tool example.
+ *
+ * Reads the specified file from HDFS and outputs to stdout.
+ *
+ * Usage: cat /<path-to-file>
+ *
+ * Example: cat /dir/file
+ *
+ * @param path-to-file Absolute path to the file to read.
+ *
+ **/
+
+#include "hdfspp/hdfspp.h"
+#include "common/hdfs_configuration.h"
+#include "common/configuration_loader.h"
+
+#include <google/protobuf/stubs/common.h>
+
+const std::size_t BUF_SIZE = 1048576; //1 MB
+static char input_buffer[BUF_SIZE];
+
+int main(int argc, char *argv[]) {
+ if (argc != 2) {
+ std::cerr << "usage: cat /<path-to-file>" << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ std::string path = argv[1];
+
+ hdfs::Options options;
+ //Setting the config path to the default: "$HADOOP_CONF_DIR" or "/etc/hadoop/conf"
+ hdfs::ConfigurationLoader loader;
+ //Loading default config files core-site.xml and hdfs-site.xml from the config path
+ hdfs::optional<hdfs::HdfsConfiguration> config = loader.LoadDefaultResources<hdfs::HdfsConfiguration>();
+ //TODO: HDFS-9539 - after this is resolved, valid config will always be returned.
+ if(config){
+ //Loading options from the config
+ options = config->GetOptions();
+ }
+ hdfs::IoService * io_service = hdfs::IoService::New();
+ //Wrapping fs into a shared pointer to guarantee deletion
+ std::shared_ptr<hdfs::FileSystem> fs(hdfs::FileSystem::New(io_service, "", options));
+ if (!fs) {
+ std::cerr << "Could not connect the file system." << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ hdfs::Status status = fs->ConnectToDefaultFs();
+ if (!status.ok()) {
+ if(!options.defaultFS.get_host().empty()){
+ std::cerr << "Error connecting to " << options.defaultFS << ". " << status.ToString() << std::endl;
+ } else {
+ std::cerr << "Error connecting to the cluster: defaultFS is empty. " << status.ToString() << std::endl;
+ }
+ exit(EXIT_FAILURE);
+ }
+
+ hdfs::FileHandle *file_raw = nullptr;
+ status = fs->Open(path, &file_raw);
+ if (!status.ok()) {
+ std::cerr << "Could not open file " << path << ". " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ //wrapping file_raw into a unique pointer to guarantee deletion
+ std::unique_ptr<hdfs::FileHandle> file(file_raw);
+
+ ssize_t total_bytes_read = 0;
+ size_t last_bytes_read = 0;
+
+ do{
+ //Reading file chunks
+ status = file->Read(input_buffer, sizeof(input_buffer), &last_bytes_read);
+ if(status.ok()) {
+ //Writing file chunks to stdout
+ fwrite(input_buffer, last_bytes_read, 1, stdout);
+ total_bytes_read += last_bytes_read;
+ } else {
+ if(status.is_invalid_offset()){
+ //Reached the end of the file
+ break;
+ } else {
+ std::cerr << "Error reading the file: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ }
+ } while (last_bytes_read > 0);
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/connect_cancel/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/connect_cancel/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/connect_cancel/CMakeLists.txt
new file mode 100644
index 0000000..6ee44cc
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/connect_cancel/CMakeLists.txt
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Default LIBHDFSPP_DIR to the default install location. You can override
+# it by add -DLIBHDFSPP_DIR=... to your cmake invocation
+set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX})
+
+include_directories( ${LIBHDFSPP_DIR}/include )
+link_directories( ${LIBHDFSPP_DIR}/lib )
+
+add_executable(connect_cancel connect_cancel.cc)
+target_link_libraries(connect_cancel hdfspp)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/connect_cancel/connect_cancel.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/connect_cancel/connect_cancel.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/connect_cancel/connect_cancel.cc
new file mode 100644
index 0000000..46eef9d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/connect_cancel/connect_cancel.cc
@@ -0,0 +1,154 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+
+#include "hdfspp/hdfspp.h"
+#include "common/hdfs_configuration.h"
+#include "common/configuration_loader.h"
+
+#include <google/protobuf/stubs/common.h>
+
+#include <signal.h>
+#include <unistd.h>
+
+#include <thread>
+#include <iostream>
+
+// Simple example of how to cancel an async connect call.
+// Here Control-C (SIGINT) is caught in order to invoke the FS level cancel and
+// properly tear down the process. Valgrind should show no leaked memory on exit
+// when cancel has been called. URI parsing code is omitted and defaultFs from
+// /etc/hadoop/conf or $HADOOP_CONF_DIR is always used.
+
+// Scoped globally to make it simple to reference from the signal handler.
+std::shared_ptr<hdfs::FileSystem> fs;
+
+const std::string catch_enter("In signal handler, going to try and cancel FileSystem::Connect.\n");
+const std::string catch_cancel("FileSystem::Cancel has been canceled in the signal handler.\n");
+const std::string catch_exit("Exiting the signal handler.\n");
+
+// Avoid IO reentrancy issues, see comments in signal handler below.
+// It's possible that the write interleaves with another write call,
+// but it won't corrupt the stack or heap.
+static void sighandler_direct_stdout(const std::string &msg) {
+ ssize_t res = ::write(1 /*posix stdout FD*/, msg.data(), msg.size());
+ // In production you'd want to check res, but error handling code will
+ // need to be fairly application specific if it's going to properly
+ // avoid reentrant calls to malloc.
+ (void)res;
+}
+
+// Signal handler to make a SIGINT call cancel rather than exit().
+static void sig_catch(int val) {
+ (void)val;
+ // This is avoiding the tricky bits of signal handling, notably that the
+ // underlying string manipulation and IO functions used by the the logger
+ // are unlikely to be reentrant.
+ //
+ // Production code could mask out all logging on handler entry and enable
+ // it again on exit; here we just assume it's "good enough" and some
+ // (possibly broken) log messages are better than none.
+
+ sighandler_direct_stdout(catch_enter);
+ if(fs) {
+ // This will invoke the callback immediately with an OperationCanceled status
+ fs->CancelPendingConnect();
+ sighandler_direct_stdout(catch_cancel);
+ }
+ sighandler_direct_stdout(catch_exit);
+}
+
+
+int main(int arg_token_count, const char **args) {
+ (void)args;
+ if(arg_token_count != 1) {
+ std::cerr << "usage: ./connect_cancel";
+ google::protobuf::ShutdownProtobufLibrary();
+ exit(EXIT_FAILURE);
+ }
+
+ // Register signal handle to asynchronously invoke cancel from outside the main thread.
+ signal(SIGINT, sig_catch);
+
+ // Generic setup/config code much like the other examples.
+ hdfs::Options options;
+ //Setting the config path to the default: "$HADOOP_CONF_DIR" or "/etc/hadoop/conf"
+ hdfs::ConfigurationLoader loader;
+ //Loading default config files core-site.xml and hdfs-site.xml from the config path
+ hdfs::optional<hdfs::HdfsConfiguration> config = loader.LoadDefaultResources<hdfs::HdfsConfiguration>();
+ //TODO: HDFS-9539 - after this is resolved, valid config will always be returned.
+ if(config){
+ //Loading options from the config
+ options = config->GetOptions();
+ }
+
+
+ // Start an IoService and some worker threads
+ std::shared_ptr<hdfs::IoService> service = hdfs::IoService::MakeShared();
+ if(nullptr == service) {
+ std::cerr << "Unable to create IoService" << std::endl;
+ fs.reset();
+ // Nasty hack to clean up for valgrind since we don't have the C++17 optional<T>::reset method
+ config = decltype(config)();
+ google::protobuf::ShutdownProtobufLibrary();
+ exit(EXIT_FAILURE);
+ }
+
+ unsigned int worker_count = service->InitDefaultWorkers();
+ if(worker_count < 1) {
+ std::cerr << "Unable to create IoService worker threads";
+ fs.reset();
+ service->Stop();
+ config = decltype(config)();
+ google::protobuf::ShutdownProtobufLibrary();
+ exit(EXIT_FAILURE);
+ }
+
+ // Set up and connect to the FileSystem
+ fs.reset(hdfs::FileSystem::New(service, "", options));
+ if(nullptr == fs) {
+ std::cerr << "Unable to create FileSystem" << std::endl;
+ fs.reset();
+ service->Stop();
+ config = decltype(config)();
+ google::protobuf::ShutdownProtobufLibrary();
+ exit(EXIT_FAILURE);
+ }
+
+ hdfs::Status status = fs->ConnectToDefaultFs();
+ if (!status.ok()) {
+ if(!options.defaultFS.get_host().empty()){
+ std::cerr << "Error connecting to " << options.defaultFS << ". " << status.ToString() << std::endl;
+ } else {
+ std::cerr << "Error connecting to the cluster: defaultFS is empty. " << status.ToString() << std::endl;
+ }
+ fs.reset();
+ service->Stop();
+ config = decltype(config)();
+ google::protobuf::ShutdownProtobufLibrary();
+ exit(EXIT_FAILURE);
+ }
+
+ fs.reset();
+ service->Stop();
+ config = decltype(config)();
+ google::protobuf::ShutdownProtobufLibrary();
+
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/find/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/find/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/find/CMakeLists.txt
new file mode 100644
index 0000000..e894e58
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/find/CMakeLists.txt
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Default LIBHDFSPP_DIR to the default install location. You can override
+# it by add -DLIBHDFSPP_DIR=... to your cmake invocation
+set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX})
+
+include_directories( ${LIBHDFSPP_DIR}/include )
+link_directories( ${LIBHDFSPP_DIR}/lib )
+
+add_executable(find_cc find.cc)
+target_link_libraries(find_cc hdfspp)
+
+# Several examples in different languages need to produce executables with
+# same names. To allow executables with same names we keep their CMake
+# names different, but specify their executable names as follows:
+set_target_properties( find_cc
+ PROPERTIES
+ OUTPUT_NAME "find"
+)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/find/find.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/find/find.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/find/find.cc
new file mode 100644
index 0000000..21a731b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/find/find.cc
@@ -0,0 +1,162 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+/**
+ * A parallel find tool example.
+ *
+ * Finds all files matching the specified name recursively starting from the
+ * specified directory and prints their filepaths. Works either synchronously
+ * or asynchronously.
+ *
+ * Usage: find /<path-to-file> <file-name> <use_async>
+ *
+ * Example: find /dir?/tree* some?file*name 1
+ *
+ * @param path-to-file Absolute path at which to begin search, can have wild
+ * cards and must be non-blank
+ * @param file-name Name to find, can have wild cards and must be non-blank
+ * @param use_async If set to 1 it prints out results asynchronously as
+ * they arrive. If set to 0 results are printed in one
+ * big chunk when it becomes available.
+ *
+ **/
+
+#include "hdfspp/hdfspp.h"
+#include "common/hdfs_configuration.h"
+#include "common/configuration_loader.h"
+
+#include <google/protobuf/stubs/common.h>
+#include <future>
+
+void SyncFind(std::shared_ptr<hdfs::FileSystem> fs, const std::string &path, const std::string &name){
+ std::vector<hdfs::StatInfo> results;
+ //Synchronous call to Find
+ hdfs::Status stat = fs->Find(path, name, hdfs::FileSystem::GetDefaultFindMaxDepth(), &results);
+
+ if (!stat.ok()) {
+ std::cerr << "Error: " << stat.ToString() << std::endl;
+ }
+
+ if(results.empty()){
+ std::cout << "Nothing Found" << std::endl;
+ } else {
+ //Printing out the results
+ for (hdfs::StatInfo const& si : results) {
+ std::cout << si.full_path << std::endl;
+ }
+ }
+}
+
+void AsyncFind(std::shared_ptr<hdfs::FileSystem> fs, const std::string &path, const std::string &name){
+ std::promise<void> promise;
+ std::future<void> future(promise.get_future());
+ bool something_found = false;
+ hdfs::Status status = hdfs::Status::OK();
+
+ /**
+ * Keep requesting more until we get the entire listing. Set the promise
+ * when we have the entire listing to stop.
+ *
+ * Find guarantees that the handler will only be called once at a time,
+ * so we do not need any locking here
+ */
+ auto handler = [&promise, &status, &something_found]
+ (const hdfs::Status &s, const std::vector<hdfs::StatInfo> & si, bool has_more_results) -> bool {
+ //Print result chunks as they arrive
+ if(!si.empty()) {
+ something_found = true;
+ for (hdfs::StatInfo const& s : si) {
+ std::cout << s.full_path << std::endl;
+ }
+ }
+ if(!s.ok() && status.ok()){
+ //We make sure we set 'status' only on the first error.
+ status = s;
+ }
+ if (!has_more_results) {
+ promise.set_value(); //set promise
+ return false; //request stop sending results
+ }
+ return true; //request more results
+ };
+
+ //Asynchronous call to Find
+ fs->Find(path, name, hdfs::FileSystem::GetDefaultFindMaxDepth(), handler);
+
+ //block until promise is set
+ future.get();
+ if(!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ }
+ if(!something_found){
+ std::cout << "Nothing Found" << std::endl;
+ }
+}
+
+int main(int argc, char *argv[]) {
+ if (argc != 4) {
+ std::cerr << "usage: find /<path-to-file> <file-name> <use_async>" << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::string path = argv[1];
+ std::string name = argv[2];
+ bool use_async = (std::stoi(argv[3]) != 0);
+
+ hdfs::Options options;
+ //Setting the config path to the default: "$HADOOP_CONF_DIR" or "/etc/hadoop/conf"
+ hdfs::ConfigurationLoader loader;
+ //Loading default config files core-site.xml and hdfs-site.xml from the config path
+ hdfs::optional<hdfs::HdfsConfiguration> config = loader.LoadDefaultResources<hdfs::HdfsConfiguration>();
+ //TODO: HDFS-9539 - after this is resolved, valid config will always be returned.
+ if(config){
+ //Loading options from the config
+ options = config->GetOptions();
+ }
+ //TODO: HDFS-9539 - until then we increase the time-out to allow all recursive async calls to finish
+ options.rpc_timeout = std::numeric_limits<int>::max();
+ hdfs::IoService * io_service = hdfs::IoService::New();
+ //Wrapping fs into a unique pointer to guarantee deletion
+ std::shared_ptr<hdfs::FileSystem> fs(hdfs::FileSystem::New(io_service, "", options));
+ if (!fs) {
+ std::cerr << "Could not connect the file system." << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ hdfs::Status status = fs->ConnectToDefaultFs();
+ if (!status.ok()) {
+ if(!options.defaultFS.get_host().empty()){
+ std::cerr << "Error connecting to " << options.defaultFS << ". " << status.ToString() << std::endl;
+ } else {
+ std::cerr << "Error connecting to the cluster: defaultFS is empty. " << status.ToString() << std::endl;
+ }
+ exit(EXIT_FAILURE);
+ }
+
+ if (use_async){
+ //Example of Async find
+ AsyncFind(fs, path, name);
+ } else {
+ //Example of Sync find
+ SyncFind(fs, path, name);
+ }
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/gendirs/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/gendirs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/gendirs/CMakeLists.txt
new file mode 100644
index 0000000..e25f425
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/gendirs/CMakeLists.txt
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Default LIBHDFSPP_DIR to the default install location. You can override
+# it by add -DLIBHDFSPP_DIR=... to your cmake invocation
+set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX})
+
+include_directories( ${LIBHDFSPP_DIR}/include )
+link_directories( ${LIBHDFSPP_DIR}/lib )
+
+add_executable(gendirs_cc gendirs.cc)
+target_link_libraries(gendirs_cc hdfspp)
+
+# Several examples in different languages need to produce executables with
+# same names. To allow executables with same names we keep their CMake
+# names different, but specify their executable names as follows:
+set_target_properties( gendirs_cc
+ PROPERTIES
+ OUTPUT_NAME "gendirs"
+)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/gendirs/gendirs.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/gendirs/gendirs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/gendirs/gendirs.cc
new file mode 100644
index 0000000..c90abbd
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/gendirs/gendirs.cc
@@ -0,0 +1,144 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+/**
+ * A recursive directory generator tool.
+ *
+ * Generates a directory tree with specified depth and fanout starting from
+ * a given path. Generation is asynchronous.
+ *
+ * Usage: gendirs /<path-to-dir> <depth> <fanout>
+ *
+ * Example: gendirs /dir0 3 10
+ *
+ * @param path-to-dir Absolute path to the directory tree root where the
+ * directory tree will be generated
+ * @param depth Depth of the directory tree (number of levels from
+ * root to leaves)
+ * @param fanout Fanout of each directory (number of sub-directories to
+ * be created inside each directory except leaf directories)
+ *
+ **/
+
+#include "hdfspp/hdfspp.h"
+#include "common/hdfs_configuration.h"
+#include "common/configuration_loader.h"
+
+#include <google/protobuf/stubs/common.h>
+#include <future>
+
+#define DEFAULT_PERMISSIONS 0755
+
+void GenerateDirectories (std::shared_ptr<hdfs::FileSystem> fs, int depth, int level, int fanout, std::string path, std::vector<std::future<hdfs::Status>> & futures) {
+ //Level contains our current depth in the directory tree
+ if(level < depth) {
+ for(int i = 0; i < fanout; i++){
+ //Recursive calls to cover all possible paths from the root to the leave nodes
+ GenerateDirectories(fs, depth, level+1, fanout, path + "dir" + std::to_string(i) + "/", futures);
+ }
+ } else {
+ //We have reached the leaf nodes and now start making calls to create directories
+ //We make a promise which will be set when the call finishes and executes our handler
+ auto callstate = std::make_shared<std::promise<hdfs::Status>>();
+ //Extract a future from this promise
+ std::future<hdfs::Status> future(callstate->get_future());
+ //Save this future to the vector of futures which will be used to wait on all promises
+ //after the whole recursion is done
+ futures.push_back(std::move(future));
+ //Create a handler that will be executed when Mkdirs is done
+ auto handler = [callstate](const hdfs::Status &s) {
+ callstate->set_value(s);
+ };
+ //Asynchronous call to create this directory along with all missing parent directories
+ fs->Mkdirs(path, DEFAULT_PERMISSIONS, true, handler);
+ }
+}
+
+int main(int argc, char *argv[]) {
+ if (argc != 4) {
+ std::cerr << "usage: gendirs /<path-to-dir> <depth> <fanout>" << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::string path = argv[1];
+ int depth = std::stoi(argv[2]);
+ int fanout = std::stoi(argv[3]);
+
+ hdfs::Options options;
+ //Setting the config path to the default: "$HADOOP_CONF_DIR" or "/etc/hadoop/conf"
+ hdfs::ConfigurationLoader loader;
+ //Loading default config files core-site.xml and hdfs-site.xml from the config path
+ hdfs::optional<hdfs::HdfsConfiguration> config = loader.LoadDefaultResources<hdfs::HdfsConfiguration>();
+ //TODO: HDFS-9539 - after this is resolved, valid config will always be returned.
+ if(config){
+ //Loading options from the config
+ options = config->GetOptions();
+ }
+ //TODO: HDFS-9539 - until then we increase the time-out to allow all recursive async calls to finish
+ options.rpc_timeout = std::numeric_limits<int>::max();
+ hdfs::IoService * io_service = hdfs::IoService::New();
+ //Wrapping fs into a unique pointer to guarantee deletion
+ std::shared_ptr<hdfs::FileSystem> fs(hdfs::FileSystem::New(io_service, "", options));
+ if (!fs) {
+ std::cerr << "Could not connect the file system." << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ hdfs::Status status = fs->ConnectToDefaultFs();
+ if (!status.ok()) {
+ if(!options.defaultFS.get_host().empty()){
+ std::cerr << "Error connecting to " << options.defaultFS << ". " << status.ToString() << std::endl;
+ } else {
+ std::cerr << "Error connecting to the cluster: defaultFS is empty. " << status.ToString() << std::endl;
+ }
+ exit(EXIT_FAILURE);
+ }
+
+ /**
+ * We do not want the recursion to block on anything, therefore we will be
+ * making asynchronous calls recursively, and then just waiting for all
+ * the calls to finish.
+ *
+ * This array of futures will be populated by the recursive function below.
+ * Each new asynchronous Mkdirs call will add a future to this vector, and will
+ * create a promise, which will only be set when the call was completed and
+ * processed. After the whole recursion is complete we will need to wait until
+ * all promises are set before we can exit.
+ **/
+ std::vector<std::future<hdfs::Status>> futures;
+
+ GenerateDirectories(fs, depth, 0, fanout, path + "/", futures);
+
+ /**
+ * We are waiting here until all promises are set, and checking whether
+ * the returned statuses contained any errors.
+ **/
+ for(std::future<hdfs::Status> &fs : futures){
+ hdfs::Status status = fs.get();
+ if (!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ std::cout << "All done!" << std::endl;
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/CMakeLists.txt
deleted file mode 100644
index 8893f03..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/CMakeLists.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-add_subdirectory(cat)
-add_subdirectory(gendirs)
-add_subdirectory(find)
-add_subdirectory(connect_cancel)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/cat/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/cat/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/cat/CMakeLists.txt
deleted file mode 100644
index 9ec3332..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/cat/CMakeLists.txt
+++ /dev/null
@@ -1,35 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Default LIBHDFSPP_DIR to the default install location. You can override
-# it by add -DLIBHDFSPP_DIR=... to your cmake invocation
-set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX})
-
-include_directories( ${LIBHDFSPP_DIR}/include )
-link_directories( ${LIBHDFSPP_DIR}/lib )
-
-add_executable(cat_cpp cat.cpp)
-target_link_libraries(cat_cpp hdfspp)
-
-# Several examples in different languages need to produce executables with
-# same names. To allow executables with same names we keep their CMake
-# names different, but specify their executable names as follows:
-set_target_properties( cat_cpp
- PROPERTIES
- OUTPUT_NAME "cat"
-)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/cat/cat.cpp
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/cat/cat.cpp b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/cat/cat.cpp
deleted file mode 100644
index 17626ea..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/cat/cat.cpp
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-*/
-
-/**
- * Unix-like cat tool example.
- *
- * Reads the specified file from HDFS and outputs to stdout.
- *
- * Usage: cat /<path-to-file>
- *
- * Example: cat /dir/file
- *
- * @param path-to-file Absolute path to the file to read.
- *
- **/
-
-#include "hdfspp/hdfspp.h"
-#include "common/hdfs_configuration.h"
-#include "common/configuration_loader.h"
-
-#include <google/protobuf/stubs/common.h>
-
-#define BUF_SIZE 4096
-
-int main(int argc, char *argv[]) {
- if (argc != 2) {
- std::cerr << "usage: cat /<path-to-file>" << std::endl;
- exit(EXIT_FAILURE);
- }
- std::string path = argv[1];
-
- hdfs::Options options;
- //Setting the config path to the default: "$HADOOP_CONF_DIR" or "/etc/hadoop/conf"
- hdfs::ConfigurationLoader loader;
- //Loading default config files core-site.xml and hdfs-site.xml from the config path
- hdfs::optional<hdfs::HdfsConfiguration> config = loader.LoadDefaultResources<hdfs::HdfsConfiguration>();
- //TODO: HDFS-9539 - after this is resolved, valid config will always be returned.
- if(config){
- //Loading options from the config
- options = config->GetOptions();
- }
- hdfs::IoService * io_service = hdfs::IoService::New();
- //Wrapping fs into a shared pointer to guarantee deletion
- std::shared_ptr<hdfs::FileSystem> fs(hdfs::FileSystem::New(io_service, "", options));
- if (!fs) {
- std::cerr << "Could not connect the file system." << std::endl;
- exit(EXIT_FAILURE);
- }
- hdfs::Status status = fs->ConnectToDefaultFs();
- if (!status.ok()) {
- if(!options.defaultFS.get_host().empty()){
- std::cerr << "Error connecting to " << options.defaultFS << ". " << status.ToString() << std::endl;
- } else {
- std::cerr << "Error connecting to the cluster: defaultFS is empty. " << status.ToString() << std::endl;
- }
- exit(EXIT_FAILURE);
- }
-
- hdfs::FileHandle *file_raw = nullptr;
- status = fs->Open(path, &file_raw);
- if (!status.ok()) {
- std::cerr << "Could not open file " << path << ". " << status.ToString() << std::endl;
- exit(EXIT_FAILURE);
- }
- //wrapping file_raw into a unique pointer to guarantee deletion
- std::unique_ptr<hdfs::FileHandle> file(file_raw);
-
- char input_buffer[BUF_SIZE];
- ssize_t total_bytes_read = 0;
- size_t last_bytes_read = 0;
-
- do{
- //Reading file chunks
- status = file->Read(input_buffer, sizeof(input_buffer), &last_bytes_read);
- if(status.ok()) {
- //Writing file chunks to stdout
- fwrite(input_buffer, last_bytes_read, 1, stdout);
- total_bytes_read += last_bytes_read;
- } else {
- if(status.is_invalid_offset()){
- //Reached the end of the file
- break;
- } else {
- std::cerr << "Error reading the file: " << status.ToString() << std::endl;
- exit(EXIT_FAILURE);
- }
- }
- } while (last_bytes_read > 0);
-
- // Clean up static data and prevent valgrind memory leaks
- google::protobuf::ShutdownProtobufLibrary();
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/connect_cancel/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/connect_cancel/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/connect_cancel/CMakeLists.txt
deleted file mode 100644
index a029a99..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/connect_cancel/CMakeLists.txt
+++ /dev/null
@@ -1,29 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Default LIBHDFSPP_DIR to the default install location. You can override
-# it by add -DLIBHDFSPP_DIR=... to your cmake invocation
-set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX})
-
-include_directories( ${LIBHDFSPP_DIR}/include )
-link_directories( ${LIBHDFSPP_DIR}/lib )
-
-add_executable(connect_cancel connect_cancel.cc)
-target_link_libraries(connect_cancel hdfspp)
-
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/connect_cancel/connect_cancel.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/connect_cancel/connect_cancel.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/connect_cancel/connect_cancel.cc
deleted file mode 100644
index 7e5daa9..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/connect_cancel/connect_cancel.cc
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-*/
-
-
-#include "hdfspp/hdfspp.h"
-#include "common/hdfs_configuration.h"
-#include "common/configuration_loader.h"
-
-#include <google/protobuf/stubs/common.h>
-
-#include <signal.h>
-#include <unistd.h>
-
-#include <thread>
-#include <iostream>
-
-// Simple example of how to cancel an async connect call.
-// Here Control-C (SIGINT) is caught in order to invoke the FS level cancel and
-// properly tear down the process. Valgrind should show no leaked memory on exit
-// when cancel has been called. URI parsing code is omitted and defaultFs from
-// /etc/hadoop/conf or $HADOOP_CONF_DIR is always used.
-
-// Scoped globally to make it simple to reference from the signal handler.
-std::shared_ptr<hdfs::FileSystem> fs;
-
-const std::string catch_enter("In signal handler, going to try and cancel FileSystem::Connect.\n");
-const std::string catch_cancel("FileSystem::Cancel has been canceled in the signal handler.\n");
-const std::string catch_exit("Exiting the signal handler.\n");
-
-// Avoid IO reentrancy issues, see comments in signal handler below.
-// It's possible that the write interleaves with another write call,
-// but it won't corrupt the stack or heap.
-static void sighandler_direct_stdout(const std::string &msg) {
- ssize_t res = ::write(1 /*posix stdout FD*/, msg.data(), msg.size());
- // In production you'd want to check res, but error handling code will
- // need to be fairly application specific if it's going to properly
- // avoid reentrant calls to malloc.
- (void)res;
-}
-
-// Signal handler to make a SIGINT call cancel rather than exit().
-static void sig_catch(int val) {
- (void)val;
- // This is avoiding the tricky bits of signal handling, notably that the
- // underlying string manipulation and IO functions used by the the logger
- // are unlikely to be reentrant.
- //
- // Production code could mask out all logging on handler entry and enable
- // it again on exit; here we just assume it's "good enough" and some
- // (possibly broken) log messages are better than none.
-
- sighandler_direct_stdout(catch_enter);
- if(fs) {
- // This will invoke the callback immediately with an OperationCanceled status
- fs->CancelPendingConnect();
- sighandler_direct_stdout(catch_cancel);
- }
- sighandler_direct_stdout(catch_exit);
-}
-
-
-int main(int arg_token_count, const char **args) {
- (void)args;
- if(arg_token_count != 1) {
- std::cerr << "usage: ./connect_cancel";
- google::protobuf::ShutdownProtobufLibrary();
- exit(EXIT_FAILURE);
- }
-
- // Register signal handle to asynchronously invoke cancel from outside the main thread.
- signal(SIGINT, sig_catch);
-
- // Generic setup/config code much like the other examples.
- hdfs::Options options;
- //Setting the config path to the default: "$HADOOP_CONF_DIR" or "/etc/hadoop/conf"
- hdfs::ConfigurationLoader loader;
- //Loading default config files core-site.xml and hdfs-site.xml from the config path
- hdfs::optional<hdfs::HdfsConfiguration> config = loader.LoadDefaultResources<hdfs::HdfsConfiguration>();
- //TODO: HDFS-9539 - after this is resolved, valid config will always be returned.
- if(config){
- //Loading options from the config
- options = config->GetOptions();
- }
-
-
- // Start an IoService and some worker threads
- std::shared_ptr<hdfs::IoService> service = hdfs::IoService::MakeShared();
- if(nullptr == service) {
- std::cerr << "Unable to create IoService" << std::endl;
- fs.reset();
- // Nasty hack to clean up for valgrind since we don't have the C++17 optional<T>::reset method
- config = decltype(config)();
- google::protobuf::ShutdownProtobufLibrary();
- exit(EXIT_FAILURE);
- }
-
- unsigned int worker_count = service->InitDefaultWorkers();
- if(worker_count < 1) {
- std::cerr << "Unable to create IoService worker threads";
- fs.reset();
- service->Stop();
- config = decltype(config)();
- google::protobuf::ShutdownProtobufLibrary();
- exit(EXIT_FAILURE);
- }
-
- // Set up and connect to the FileSystem
- fs.reset(hdfs::FileSystem::New(service, "", options));
- if(nullptr == fs) {
- std::cerr << "Unable to create FileSystem" << std::endl;
- fs.reset();
- service->Stop();
- config = decltype(config)();
- google::protobuf::ShutdownProtobufLibrary();
- exit(EXIT_FAILURE);
- }
-
- hdfs::Status status = fs->ConnectToDefaultFs();
- if (!status.ok()) {
- if(!options.defaultFS.get_host().empty()){
- std::cerr << "Error connecting to " << options.defaultFS << ". " << status.ToString() << std::endl;
- } else {
- std::cerr << "Error connecting to the cluster: defaultFS is empty. " << status.ToString() << std::endl;
- }
- fs.reset();
- service->Stop();
- config = decltype(config)();
- google::protobuf::ShutdownProtobufLibrary();
- exit(EXIT_FAILURE);
- }
-
- fs.reset();
- service->Stop();
- config = decltype(config)();
- google::protobuf::ShutdownProtobufLibrary();
-
- return 0;
-}
-
-
-
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/find/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/find/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/find/CMakeLists.txt
deleted file mode 100644
index 7ae27c5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/find/CMakeLists.txt
+++ /dev/null
@@ -1,35 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Default LIBHDFSPP_DIR to the default install location. You can override
-# it by add -DLIBHDFSPP_DIR=... to your cmake invocation
-set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX})
-
-include_directories( ${LIBHDFSPP_DIR}/include )
-link_directories( ${LIBHDFSPP_DIR}/lib )
-
-add_executable(find_cpp find.cpp)
-target_link_libraries(find_cpp hdfspp)
-
-# Several examples in different languages need to produce executables with
-# same names. To allow executables with same names we keep their CMake
-# names different, but specify their executable names as follows:
-set_target_properties( find_cpp
- PROPERTIES
- OUTPUT_NAME "find"
-)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/find/find.cpp
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/find/find.cpp b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/find/find.cpp
deleted file mode 100644
index 21a731b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/find/find.cpp
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-*/
-
-/**
- * A parallel find tool example.
- *
- * Finds all files matching the specified name recursively starting from the
- * specified directory and prints their filepaths. Works either synchronously
- * or asynchronously.
- *
- * Usage: find /<path-to-file> <file-name> <use_async>
- *
- * Example: find /dir?/tree* some?file*name 1
- *
- * @param path-to-file Absolute path at which to begin search, can have wild
- * cards and must be non-blank
- * @param file-name Name to find, can have wild cards and must be non-blank
- * @param use_async If set to 1 it prints out results asynchronously as
- * they arrive. If set to 0 results are printed in one
- * big chunk when it becomes available.
- *
- **/
-
-#include "hdfspp/hdfspp.h"
-#include "common/hdfs_configuration.h"
-#include "common/configuration_loader.h"
-
-#include <google/protobuf/stubs/common.h>
-#include <future>
-
-void SyncFind(std::shared_ptr<hdfs::FileSystem> fs, const std::string &path, const std::string &name){
- std::vector<hdfs::StatInfo> results;
- //Synchronous call to Find
- hdfs::Status stat = fs->Find(path, name, hdfs::FileSystem::GetDefaultFindMaxDepth(), &results);
-
- if (!stat.ok()) {
- std::cerr << "Error: " << stat.ToString() << std::endl;
- }
-
- if(results.empty()){
- std::cout << "Nothing Found" << std::endl;
- } else {
- //Printing out the results
- for (hdfs::StatInfo const& si : results) {
- std::cout << si.full_path << std::endl;
- }
- }
-}
-
-void AsyncFind(std::shared_ptr<hdfs::FileSystem> fs, const std::string &path, const std::string &name){
- std::promise<void> promise;
- std::future<void> future(promise.get_future());
- bool something_found = false;
- hdfs::Status status = hdfs::Status::OK();
-
- /**
- * Keep requesting more until we get the entire listing. Set the promise
- * when we have the entire listing to stop.
- *
- * Find guarantees that the handler will only be called once at a time,
- * so we do not need any locking here
- */
- auto handler = [&promise, &status, &something_found]
- (const hdfs::Status &s, const std::vector<hdfs::StatInfo> & si, bool has_more_results) -> bool {
- //Print result chunks as they arrive
- if(!si.empty()) {
- something_found = true;
- for (hdfs::StatInfo const& s : si) {
- std::cout << s.full_path << std::endl;
- }
- }
- if(!s.ok() && status.ok()){
- //We make sure we set 'status' only on the first error.
- status = s;
- }
- if (!has_more_results) {
- promise.set_value(); //set promise
- return false; //request stop sending results
- }
- return true; //request more results
- };
-
- //Asynchronous call to Find
- fs->Find(path, name, hdfs::FileSystem::GetDefaultFindMaxDepth(), handler);
-
- //block until promise is set
- future.get();
- if(!status.ok()) {
- std::cerr << "Error: " << status.ToString() << std::endl;
- }
- if(!something_found){
- std::cout << "Nothing Found" << std::endl;
- }
-}
-
-int main(int argc, char *argv[]) {
- if (argc != 4) {
- std::cerr << "usage: find /<path-to-file> <file-name> <use_async>" << std::endl;
- exit(EXIT_FAILURE);
- }
-
- std::string path = argv[1];
- std::string name = argv[2];
- bool use_async = (std::stoi(argv[3]) != 0);
-
- hdfs::Options options;
- //Setting the config path to the default: "$HADOOP_CONF_DIR" or "/etc/hadoop/conf"
- hdfs::ConfigurationLoader loader;
- //Loading default config files core-site.xml and hdfs-site.xml from the config path
- hdfs::optional<hdfs::HdfsConfiguration> config = loader.LoadDefaultResources<hdfs::HdfsConfiguration>();
- //TODO: HDFS-9539 - after this is resolved, valid config will always be returned.
- if(config){
- //Loading options from the config
- options = config->GetOptions();
- }
- //TODO: HDFS-9539 - until then we increase the time-out to allow all recursive async calls to finish
- options.rpc_timeout = std::numeric_limits<int>::max();
- hdfs::IoService * io_service = hdfs::IoService::New();
- //Wrapping fs into a unique pointer to guarantee deletion
- std::shared_ptr<hdfs::FileSystem> fs(hdfs::FileSystem::New(io_service, "", options));
- if (!fs) {
- std::cerr << "Could not connect the file system." << std::endl;
- exit(EXIT_FAILURE);
- }
- hdfs::Status status = fs->ConnectToDefaultFs();
- if (!status.ok()) {
- if(!options.defaultFS.get_host().empty()){
- std::cerr << "Error connecting to " << options.defaultFS << ". " << status.ToString() << std::endl;
- } else {
- std::cerr << "Error connecting to the cluster: defaultFS is empty. " << status.ToString() << std::endl;
- }
- exit(EXIT_FAILURE);
- }
-
- if (use_async){
- //Example of Async find
- AsyncFind(fs, path, name);
- } else {
- //Example of Sync find
- SyncFind(fs, path, name);
- }
-
- // Clean up static data and prevent valgrind memory leaks
- google::protobuf::ShutdownProtobufLibrary();
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/gendirs/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/gendirs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/gendirs/CMakeLists.txt
deleted file mode 100644
index 90cddf0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/gendirs/CMakeLists.txt
+++ /dev/null
@@ -1,35 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Default LIBHDFSPP_DIR to the default install location. You can override
-# it by add -DLIBHDFSPP_DIR=... to your cmake invocation
-set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX})
-
-include_directories( ${LIBHDFSPP_DIR}/include )
-link_directories( ${LIBHDFSPP_DIR}/lib )
-
-add_executable(gendirs_cpp gendirs.cpp)
-target_link_libraries(gendirs_cpp hdfspp)
-
-# Several examples in different languages need to produce executables with
-# same names. To allow executables with same names we keep their CMake
-# names different, but specify their executable names as follows:
-set_target_properties( gendirs_cpp
- PROPERTIES
- OUTPUT_NAME "gendirs"
-)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/gendirs/gendirs.cpp
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/gendirs/gendirs.cpp b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/gendirs/gendirs.cpp
deleted file mode 100644
index c90abbd..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/gendirs/gendirs.cpp
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-*/
-
-/**
- * A recursive directory generator tool.
- *
- * Generates a directory tree with specified depth and fanout starting from
- * a given path. Generation is asynchronous.
- *
- * Usage: gendirs /<path-to-dir> <depth> <fanout>
- *
- * Example: gendirs /dir0 3 10
- *
- * @param path-to-dir Absolute path to the directory tree root where the
- * directory tree will be generated
- * @param depth Depth of the directory tree (number of levels from
- * root to leaves)
- * @param fanout Fanout of each directory (number of sub-directories to
- * be created inside each directory except leaf directories)
- *
- **/
-
-#include "hdfspp/hdfspp.h"
-#include "common/hdfs_configuration.h"
-#include "common/configuration_loader.h"
-
-#include <google/protobuf/stubs/common.h>
-#include <future>
-
-#define DEFAULT_PERMISSIONS 0755
-
-void GenerateDirectories (std::shared_ptr<hdfs::FileSystem> fs, int depth, int level, int fanout, std::string path, std::vector<std::future<hdfs::Status>> & futures) {
- //Level contains our current depth in the directory tree
- if(level < depth) {
- for(int i = 0; i < fanout; i++){
- //Recursive calls to cover all possible paths from the root to the leave nodes
- GenerateDirectories(fs, depth, level+1, fanout, path + "dir" + std::to_string(i) + "/", futures);
- }
- } else {
- //We have reached the leaf nodes and now start making calls to create directories
- //We make a promise which will be set when the call finishes and executes our handler
- auto callstate = std::make_shared<std::promise<hdfs::Status>>();
- //Extract a future from this promise
- std::future<hdfs::Status> future(callstate->get_future());
- //Save this future to the vector of futures which will be used to wait on all promises
- //after the whole recursion is done
- futures.push_back(std::move(future));
- //Create a handler that will be executed when Mkdirs is done
- auto handler = [callstate](const hdfs::Status &s) {
- callstate->set_value(s);
- };
- //Asynchronous call to create this directory along with all missing parent directories
- fs->Mkdirs(path, DEFAULT_PERMISSIONS, true, handler);
- }
-}
-
-int main(int argc, char *argv[]) {
- if (argc != 4) {
- std::cerr << "usage: gendirs /<path-to-dir> <depth> <fanout>" << std::endl;
- exit(EXIT_FAILURE);
- }
-
- std::string path = argv[1];
- int depth = std::stoi(argv[2]);
- int fanout = std::stoi(argv[3]);
-
- hdfs::Options options;
- //Setting the config path to the default: "$HADOOP_CONF_DIR" or "/etc/hadoop/conf"
- hdfs::ConfigurationLoader loader;
- //Loading default config files core-site.xml and hdfs-site.xml from the config path
- hdfs::optional<hdfs::HdfsConfiguration> config = loader.LoadDefaultResources<hdfs::HdfsConfiguration>();
- //TODO: HDFS-9539 - after this is resolved, valid config will always be returned.
- if(config){
- //Loading options from the config
- options = config->GetOptions();
- }
- //TODO: HDFS-9539 - until then we increase the time-out to allow all recursive async calls to finish
- options.rpc_timeout = std::numeric_limits<int>::max();
- hdfs::IoService * io_service = hdfs::IoService::New();
- //Wrapping fs into a unique pointer to guarantee deletion
- std::shared_ptr<hdfs::FileSystem> fs(hdfs::FileSystem::New(io_service, "", options));
- if (!fs) {
- std::cerr << "Could not connect the file system." << std::endl;
- exit(EXIT_FAILURE);
- }
- hdfs::Status status = fs->ConnectToDefaultFs();
- if (!status.ok()) {
- if(!options.defaultFS.get_host().empty()){
- std::cerr << "Error connecting to " << options.defaultFS << ". " << status.ToString() << std::endl;
- } else {
- std::cerr << "Error connecting to the cluster: defaultFS is empty. " << status.ToString() << std::endl;
- }
- exit(EXIT_FAILURE);
- }
-
- /**
- * We do not want the recursion to block on anything, therefore we will be
- * making asynchronous calls recursively, and then just waiting for all
- * the calls to finish.
- *
- * This array of futures will be populated by the recursive function below.
- * Each new asynchronous Mkdirs call will add a future to this vector, and will
- * create a promise, which will only be set when the call was completed and
- * processed. After the whole recursion is complete we will need to wait until
- * all promises are set before we can exit.
- **/
- std::vector<std::future<hdfs::Status>> futures;
-
- GenerateDirectories(fs, depth, 0, fanout, path + "/", futures);
-
- /**
- * We are waiting here until all promises are set, and checking whether
- * the returned statuses contained any errors.
- **/
- for(std::future<hdfs::Status> &fs : futures){
- hdfs::Status status = fs.get();
- if (!status.ok()) {
- std::cerr << "Error: " << status.ToString() << std::endl;
- exit(EXIT_FAILURE);
- }
- }
-
- std::cout << "All done!" << std::endl;
-
- // Clean up static data and prevent valgrind memory leaks
- google::protobuf::ShutdownProtobufLibrary();
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/content_summary.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/content_summary.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/content_summary.h
new file mode 100644
index 0000000..6762a65
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/content_summary.h
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef HDFSPP_CONTENT_SUMMARY_H_
+#define HDFSPP_CONTENT_SUMMARY_H_
+
+#include <string>
+
+namespace hdfs {
+
+/**
+ * Content summary is assumed to be unchanging for the duration of the operation
+ */
+struct ContentSummary {
+ uint64_t length;
+ uint64_t filecount;
+ uint64_t directorycount;
+ uint64_t quota;
+ uint64_t spaceconsumed;
+ uint64_t spacequota;
+ std::string path;
+
+ ContentSummary();
+
+ //Converts ContentSummary object to std::string (hdfs_count format)
+ std::string str(bool include_quota) const;
+
+ //Converts ContentSummary object to std::string (hdfs_du format)
+ std::string str_du() const;
+};
+
+}
+
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/fsinfo.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/fsinfo.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/fsinfo.h
index bd58592..103cf13 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/fsinfo.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/fsinfo.h
@@ -18,6 +18,8 @@
#ifndef HDFSPP_FSINFO_H_
#define HDFSPP_FSINFO_H_
+#include <string>
+
namespace hdfs {
/**
@@ -26,25 +28,19 @@ namespace hdfs {
*/
struct FsInfo {
- unsigned long int capacity;
- unsigned long int used;
- unsigned long int remaining;
- unsigned long int under_replicated;
- unsigned long int corrupt_blocks;
- unsigned long int missing_blocks;
- unsigned long int missing_repl_one_blocks;
- unsigned long int blocks_in_future;
-
- FsInfo()
- : capacity(0),
- used(0),
- remaining(0),
- under_replicated(0),
- corrupt_blocks(0),
- missing_blocks(0),
- missing_repl_one_blocks(0),
- blocks_in_future(0) {
- }
+ uint64_t capacity;
+ uint64_t used;
+ uint64_t remaining;
+ uint64_t under_replicated;
+ uint64_t corrupt_blocks;
+ uint64_t missing_blocks;
+ uint64_t missing_repl_one_blocks;
+ uint64_t blocks_in_future;
+
+ FsInfo();
+
+ //Converts FsInfo object to std::string (hdfs_df format)
+ std::string str(const std::string fs_name) const;
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
index 2b15cf4..cdd3d4d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
@@ -330,6 +330,17 @@ LIBHDFS_EXTERNAL
int hdfsDeleteSnapshot(hdfsFS fs, const char* path, const char* name);
/**
+ * Renames the directory snapshot specified by path from old_name to new_name
+ *
+ * @param fs The filesystem (required)
+ * @param path Path to the snapshotted directory (must be non-blank)
+ * @param old_name Current name of the snapshot (must be non-blank)
+ * @param new_name New name of the snapshot (must be non-blank)
+ * @return 0 on success, corresponding errno on failure
+ **/
+int hdfsRenameSnapshot(hdfsFS fs, const char* path, const char* old_name, const char* new_name);
+
+/**
* Allows snapshots to be made on the specified directory
*
* @param fs The filesystem (required)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
index 46a1e61..d193b5f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
@@ -24,6 +24,7 @@
#include "hdfspp/block_location.h"
#include "hdfspp/statinfo.h"
#include "hdfspp/fsinfo.h"
+#include "hdfspp/content_summary.h"
#include <functional>
#include <memory>
@@ -276,6 +277,14 @@ class FileSystem {
virtual Status GetFileInfo(const std::string &path, StatInfo & stat_info) = 0;
/**
+ * Returns the number of directories, files and bytes under the given path
+ **/
+ virtual void
+ GetContentSummary(const std::string &path,
+ const std::function<void(const Status &, const ContentSummary &)> &handler) = 0;
+ virtual Status GetContentSummary(const std::string &path, ContentSummary & stat_info) = 0;
+
+ /**
* Retrieves the file system information as a whole, such as the total raw size of all files in the filesystem
* and the raw capacity of the filesystem
*
@@ -305,7 +314,7 @@ class FileSystem {
/**
* Returns the locations of all known blocks for the indicated file (or part of it), or an error
- * if the information clould not be found
+ * if the information could not be found
*/
virtual void GetBlockLocations(const std::string & path, uint64_t offset, uint64_t length,
const std::function<void(const Status &, std::shared_ptr<FileBlockLocation> locations)> ) = 0;
@@ -418,6 +427,18 @@ class FileSystem {
const std::string &name) = 0;
/**
+ * Renames the directory snapshot specified by path from old_name to new_name
+ *
+ * @param path Path to the snapshotted directory (must be non-blank)
+ * @param old_name Current name of the snapshot (must be non-blank)
+ * @param new_name New name of the snapshot (must be non-blank)
+ **/
+ virtual void RenameSnapshot(const std::string &path, const std::string &old_name,
+ const std::string &new_name, const std::function<void(const Status &)> &handler) = 0;
+ virtual Status RenameSnapshot(const std::string &path, const std::string &old_name,
+ const std::string &new_name) = 0;
+
+ /**
* Allows snapshots to be made on the specified directory
*
* @param path Path to the directory to be made snapshottable (must be non-empty)
@@ -454,6 +475,8 @@ class FileSystem {
virtual void SetFsEventCallback(fs_event_callback callback) = 0;
virtual Options get_options() = 0;
+
+ virtual std::string get_cluster_name() = 0;
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h
index e077dda..1da1f18 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h
@@ -18,6 +18,8 @@
#ifndef HDFSPP_STATINFO_H_
#define HDFSPP_STATINFO_H_
+#include <string>
+
namespace hdfs {
/**
@@ -31,31 +33,25 @@ struct StatInfo {
IS_SYMLINK = 3
};
- int file_type;
- ::std::string path;
- ::std::string full_path;
- unsigned long int length;
- unsigned long int permissions; //Octal number as in POSIX permissions; e.g. 0777
- ::std::string owner;
- ::std::string group;
- unsigned long int modification_time;
- unsigned long int access_time;
- ::std::string symlink;
- unsigned int block_replication;
- unsigned long int blocksize;
- unsigned long int fileid;
- unsigned long int children_num;
- StatInfo()
- : file_type(0),
- length(0),
- permissions(0),
- modification_time(0),
- access_time(0),
- block_replication(0),
- blocksize(0),
- fileid(0),
- children_num(0) {
- }
+ int file_type;
+ std::string path;
+ std::string full_path;
+ uint64_t length;
+ uint64_t permissions; //Octal number as in POSIX permissions; e.g. 0777
+ std::string owner;
+ std::string group;
+ uint64_t modification_time;
+ uint64_t access_time;
+ std::string symlink;
+ uint32_t block_replication;
+ uint64_t blocksize;
+ uint64_t fileid;
+ uint64_t children_num;
+
+ StatInfo();
+
+ //Converts StatInfo object to std::string (hdfs_ls format)
+ std::string str() const;
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
index 8c7e34c..7e895b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
@@ -1094,6 +1094,37 @@ int hdfsDeleteSnapshot(hdfsFS fs, const char* path, const char* name) {
}
}
+
+int hdfsRenameSnapshot(hdfsFS fs, const char* path, const char* old_name, const char* new_name) {
+ try {
+ errno = 0;
+ if (!CheckSystem(fs)) {
+ return -1;
+ }
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
+ }
+ if (!old_name) {
+ return Error(Status::InvalidArgument("hdfsRenameSnapshot: argument 'old_name' cannot be NULL"));
+ }
+ if (!new_name) {
+ return Error(Status::InvalidArgument("hdfsRenameSnapshot: argument 'new_name' cannot be NULL"));
+ }
+ Status stat;
+ stat = fs->get_impl()->RenameSnapshot(*abs_path, old_name, new_name);
+ if (!stat.ok()) {
+ return Error(stat);
+ }
+ return 0;
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
+
+}
+
int hdfsAllowSnapshot(hdfsFS fs, const char* path) {
try {
errno = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/869317be/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
index bdeb068..b0b721a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
@@ -19,6 +19,6 @@ if(NEED_LINK_DL)
set(LIB_DL dl)
endif()
-add_library(common_obj OBJECT status.cc sasl_digest_md5.cc hdfs_ioservice.cc options.cc configuration.cc configuration_loader.cc hdfs_configuration.cc uri.cc util.cc retry_policy.cc cancel_tracker.cc logging.cc libhdfs_events_impl.cc auth_info.cc namenode_info.cc)
+add_library(common_obj OBJECT status.cc sasl_digest_md5.cc hdfs_ioservice.cc options.cc configuration.cc configuration_loader.cc hdfs_configuration.cc uri.cc util.cc retry_policy.cc cancel_tracker.cc logging.cc libhdfs_events_impl.cc auth_info.cc namenode_info.cc statinfo.cc fsinfo.cc content_summary.cc)
add_library(common $<TARGET_OBJECTS:common_obj> $<TARGET_OBJECTS:uriparser2_obj>)
target_link_libraries(common ${LIB_DL})
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/52] [abbrv] hadoop git commit: HDFS-11544: libhdfs++: Revert
HDFS-11544.HDFS-8707.001.patch due to invalid memory access regression. Done
by James Clampffer
Posted by jh...@apache.org.
HDFS-11544: libhdfs++: Revert HDFS-11544.HDFS-8707.001.patch due to invalid memory access regression. Done by James Clampffer
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/52445fd4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/52445fd4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/52445fd4
Branch: refs/heads/HDFS-8707
Commit: 52445fd4ee73df0bd113e257fb4435796241486e
Parents: c2386bc
Author: James Clampffer <ja...@hp.com>
Authored: Fri May 19 15:38:17 2017 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../native/libhdfspp/lib/bindings/c/hdfs.cc | 800 +++++++++----------
1 file changed, 398 insertions(+), 402 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52445fd4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
index 4e6c0f3..8c7e34c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
@@ -41,23 +41,6 @@ using namespace std::placeholders;
static constexpr tPort kDefaultPort = 8020;
-// Functions exposed through the C API and extended C API get tagged with no-op
-// defines. This is to make it really clear what's being exposed in a way that
-// won't blend into the comments. Can also be used to add attributes or disable
-// the extensions e.g. "#define HDFS_EXT_API static"
-// API declared in hdfs.h shared with libhdfs
-#define HDFS_API
-// API extension functions specific to libhdfs++ declared in hdfs_ext.h
-#define HDFS_EXT_API
-
-// Common catch-all case to get exception and return the appropriate error code
-#define CATCH_AND_REPORT catch (const std::exception & e) { return ReportException(e); } \
- catch (...) { return ReportCaughtNonException(); }
-
-
-
-
-
/* Separate the handles used by the C api from the C++ API*/
struct hdfs_internal {
hdfs_internal(FileSystem *p) : filesystem_(p), working_directory_("/") {}
@@ -96,14 +79,9 @@ struct hdfsFile_internal {
thread_local std::string errstr;
/* Fetch last error that happened in this thread */
-HDFS_EXT_API
int hdfsGetLastError(char *buf, int len) {
//No error message
if(errstr.empty()){
- if(len > 0) {
- // Null terminator if there is space
- *buf = 0;
- }
return -1;
}
@@ -126,17 +104,6 @@ int hdfsGetLastError(char *buf, int len) {
return 0;
}
-/* Clear out last error. This shouldn't need to be public if functions do it themselves */
-static void resetErrorForApiCall(const char *location = nullptr) {
- // POSIX spec says caller should be responsible for resetting errno.
- // Clearing it here just in case.
- errno = 0;
- if(location)
- errstr = std::string(location) + " has not set an error message";
- else
- errstr.clear();
-}
-
/* Event callbacks for next open calls */
thread_local std::experimental::optional<fs_event_callback> fsEventCallback;
thread_local std::experimental::optional<file_event_callback> fileEventCallback;
@@ -288,9 +255,7 @@ optional<std::string> getAbsolutePath(hdfsFS fs, const char* path) {
* C API implementations
**/
-HDFS_API
int hdfsFileIsOpenForRead(hdfsFile file) {
- resetErrorForApiCall("hdfsFileIsOpenForRead");
/* files can only be open for reads at the moment, do a quick check */
if (!CheckHandle(file)){
return 0;
@@ -298,9 +263,7 @@ int hdfsFileIsOpenForRead(hdfsFile file) {
return 1; // Update implementation when we get file writing
}
-HDFS_API
int hdfsFileIsOpenForWrite(hdfsFile file) {
- resetErrorForApiCall("hdfsFileIsOpenForWrite");
/* files can only be open for reads at the moment, so return false */
CheckHandle(file);
return -1; // Update implementation when we get file writing
@@ -313,7 +276,11 @@ int hdfsConfGetLong(const char *key, int64_t *val)
errno = 0;
hdfsBuilder builder;
return hdfsBuilderConfGetLong(&builder, key, val);
- } CATCH_AND_REPORT
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
}
hdfsFS doHdfsConnect(optional<std::string> nn, optional<tPort> port, optional<std::string> user, const Options & options) {
@@ -365,12 +332,11 @@ hdfsFS doHdfsConnect(optional<std::string> nn, optional<tPort> port, optional<st
}
}
-HDFS_EXT_API
hdfsFS hdfsAllocateFileSystem(struct hdfsBuilder *bld) {
- resetErrorForApiCall("hdfsAllocateFileSystem");
// Same idea as the first half of doHdfsConnect, but return the wrapped FS before
// connecting.
try {
+ errno = 0;
std::shared_ptr<IoService> io_service = IoService::MakeShared();
int io_thread_count = bld->config.GetOptions().io_threads_;
@@ -401,9 +367,7 @@ hdfsFS hdfsAllocateFileSystem(struct hdfsBuilder *bld) {
return nullptr;
}
-HDFS_EXT_API
int hdfsConnectAllocated(hdfsFS fs, struct hdfsBuilder *bld) {
- resetErrorForApiCall("hdfsConnectAllocated");
if(!CheckSystem(fs)) {
return ENODEV;
}
@@ -456,35 +420,25 @@ int hdfsConnectAllocated(hdfsFS fs, struct hdfsBuilder *bld) {
return 0;
}
-HDFS_API
hdfsFS hdfsConnect(const char *nn, tPort port) {
- resetErrorForApiCall("hdfsConnect");
- return doHdfsConnect(std::string(nn), port, std::string(""), Options());
+ return hdfsConnectAsUser(nn, port, "");
}
-HDFS_API
hdfsFS hdfsConnectAsUser(const char* nn, tPort port, const char *user) {
- resetErrorForApiCall("hdfsConnectAsUser");
return doHdfsConnect(std::string(nn), port, std::string(user), Options());
}
-HDFS_API
hdfsFS hdfsConnectAsUserNewInstance(const char* nn, tPort port, const char *user ) {
- resetErrorForApiCall("hdfsConnectAsUserNewInstance");
//libhdfspp always returns a new instance
return doHdfsConnect(std::string(nn), port, std::string(user), Options());
}
-HDFS_API
hdfsFS hdfsConnectNewInstance(const char* nn, tPort port) {
- resetErrorForApiCall("hdfsConnectNewInstance");
//libhdfspp always returns a new instance
- return doHdfsConnect(std::string(nn), port, std::string(""), Options());
+ return hdfsConnectAsUser(nn, port, "");
}
-HDFS_EXT_API
int hdfsCancelPendingConnection(hdfsFS fs) {
- resetErrorForApiCall("hdfsCancelPendingConnection");
// todo: stick an enum in hdfs_internal to check the connect state
if(!CheckSystem(fs)) {
return ENODEV;
@@ -504,11 +458,10 @@ int hdfsCancelPendingConnection(hdfsFS fs) {
}
}
-HDFS_API
int hdfsDisconnect(hdfsFS fs) {
- resetErrorForApiCall("hdfsDisconnect");
try
{
+ errno = 0;
if (!fs) {
ReportError(ENODEV, "Cannot disconnect null FS handle.");
return -1;
@@ -516,15 +469,18 @@ int hdfsDisconnect(hdfsFS fs) {
delete fs;
return 0;
- } CATCH_AND_REPORT
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
}
-HDFS_API
hdfsFile hdfsOpenFile(hdfsFS fs, const char *path, int flags, int bufferSize,
short replication, tSize blocksize) {
- resetErrorForApiCall("hdfsOpenFile");
try
{
+ errno = 0;
(void)flags;
(void)bufferSize;
(void)replication;
@@ -556,24 +512,26 @@ hdfsFile hdfsOpenFile(hdfsFS fs, const char *path, int flags, int bufferSize,
}
}
-HDFS_API
int hdfsCloseFile(hdfsFS fs, hdfsFile file) {
- resetErrorForApiCall("hdfsCloseFile");
try
{
+ errno = 0;
if (!CheckSystemAndHandle(fs, file)) {
return -1;
}
delete file;
return 0;
- } CATCH_AND_REPORT
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
}
-HDFS_API
char* hdfsGetWorkingDirectory(hdfsFS fs, char *buffer, size_t bufferSize) {
- resetErrorForApiCall("hdfsGetWorkingDirectory");
try
{
+ errno = 0;
if (!CheckSystem(fs)) {
return nullptr;
}
@@ -598,11 +556,10 @@ char* hdfsGetWorkingDirectory(hdfsFS fs, char *buffer, size_t bufferSize) {
}
}
-HDFS_API
int hdfsSetWorkingDirectory(hdfsFS fs, const char* path) {
- resetErrorForApiCall("hdfsSetWorkingDirectory");
try
{
+ errno = 0;
if (!CheckSystem(fs)) {
return -1;
}
@@ -618,23 +575,25 @@ int hdfsSetWorkingDirectory(hdfsFS fs, const char* path) {
}
fs->set_working_directory(withSlash);
return 0;
- } CATCH_AND_REPORT
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
}
-HDFS_API
int hdfsAvailable(hdfsFS fs, hdfsFile file) {
- resetErrorForApiCall("hdfsAvailable");
//Since we do not have read ahead implemented, return 0 if fs and file are good;
+ errno = 0;
if (!CheckSystemAndHandle(fs, file)) {
return -1;
}
return 0;
}
-HDFS_API
tOffset hdfsGetDefaultBlockSize(hdfsFS fs) {
- resetErrorForApiCall("hdfsGetDefaultBlockSize");
try {
+ errno = 0;
return fs->get_impl()->get_options().block_size;
} catch (const std::exception & e) {
ReportException(e);
@@ -645,10 +604,9 @@ tOffset hdfsGetDefaultBlockSize(hdfsFS fs) {
}
}
-HDFS_API
tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path) {
- resetErrorForApiCall("hdfsGetDefaultBlockSizeAtPath");
try {
+ errno = 0;
if (!CheckSystem(fs)) {
return -1;
}
@@ -675,53 +633,58 @@ tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path) {
}
}
-HDFS_API
int hdfsSetReplication(hdfsFS fs, const char* path, int16_t replication) {
- resetErrorForApiCall("hdfsSetReplication");
- try {
- if (!CheckSystem(fs)) {
- return -1;
- }
- const optional<std::string> abs_path = getAbsolutePath(fs, path);
- if(!abs_path) {
- return -1;
- }
- if(replication < 1){
- return Error(Status::InvalidArgument("SetReplication: argument 'replication' cannot be less than 1"));
- }
- Status stat;
- stat = fs->get_impl()->SetReplication(*abs_path, replication);
- if (!stat.ok()) {
- return Error(stat);
+ try {
+ errno = 0;
+ if (!CheckSystem(fs)) {
+ return -1;
+ }
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
+ }
+ if(replication < 1){
+ return Error(Status::InvalidArgument("SetReplication: argument 'replication' cannot be less than 1"));
+ }
+ Status stat;
+ stat = fs->get_impl()->SetReplication(*abs_path, replication);
+ if (!stat.ok()) {
+ return Error(stat);
+ }
+ return 0;
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
}
- return 0;
- } CATCH_AND_REPORT
}
-HDFS_API
int hdfsUtime(hdfsFS fs, const char* path, tTime mtime, tTime atime) {
- resetErrorForApiCall("hdfsUtime");
- try {
- if (!CheckSystem(fs)) {
- return -1;
- }
- const optional<std::string> abs_path = getAbsolutePath(fs, path);
- if(!abs_path) {
- return -1;
- }
- Status stat;
- stat = fs->get_impl()->SetTimes(*abs_path, mtime, atime);
- if (!stat.ok()) {
- return Error(stat);
+ try {
+ errno = 0;
+ if (!CheckSystem(fs)) {
+ return -1;
+ }
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
+ }
+ Status stat;
+ stat = fs->get_impl()->SetTimes(*abs_path, mtime, atime);
+ if (!stat.ok()) {
+ return Error(stat);
+ }
+ return 0;
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
}
- return 0;
- } CATCH_AND_REPORT
}
-HDFS_API
tOffset hdfsGetCapacity(hdfsFS fs) {
- resetErrorForApiCall("hdfsGetCapacity");
try {
+ errno = 0;
if (!CheckSystem(fs)) {
return -1;
}
@@ -742,10 +705,9 @@ tOffset hdfsGetCapacity(hdfsFS fs) {
}
}
-HDFS_API
tOffset hdfsGetUsed(hdfsFS fs) {
- resetErrorForApiCall("hdfsGetUsed");
try {
+ errno = 0;
if (!CheckSystem(fs)) {
return -1;
}
@@ -815,10 +777,9 @@ void StatInfoToHdfsFileInfo(hdfsFileInfo * file_info,
file_info->mLastAccess = stat_info.access_time;
}
-HDFS_API
int hdfsExists(hdfsFS fs, const char *path) {
- resetErrorForApiCall("hdfsExists");
try {
+ errno = 0;
if (!CheckSystem(fs)) {
return -1;
}
@@ -832,13 +793,16 @@ int hdfsExists(hdfsFS fs, const char *path) {
return Error(stat);
}
return 0;
- } CATCH_AND_REPORT
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
}
-HDFS_API
hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char* path) {
- resetErrorForApiCall("hdfsGetPathInfo");
try {
+ errno = 0;
if (!CheckSystem(fs)) {
return nullptr;
}
@@ -864,64 +828,61 @@ hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char* path) {
}
}
-HDFS_API
hdfsFileInfo *hdfsListDirectory(hdfsFS fs, const char* path, int *numEntries) {
- resetErrorForApiCall("hdfsListDirectory");
try {
- if (!CheckSystem(fs)) {
- *numEntries = 0;
- return nullptr;
- }
- const optional<std::string> abs_path = getAbsolutePath(fs, path);
- if(!abs_path) {
- return nullptr;
- }
- std::vector<StatInfo> stat_infos;
- Status stat = fs->get_impl()->GetListing(*abs_path, &stat_infos);
- if (!stat.ok()) {
- Error(stat);
+ errno = 0;
+ if (!CheckSystem(fs)) {
+ *numEntries = 0;
+ return nullptr;
+ }
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return nullptr;
+ }
+ std::vector<StatInfo> stat_infos;
+ Status stat = fs->get_impl()->GetListing(*abs_path, &stat_infos);
+ if (!stat.ok()) {
+ Error(stat);
+ *numEntries = 0;
+ return nullptr;
+ }
+ if(stat_infos.empty()){
+ *numEntries = 0;
+ return nullptr;
+ }
+ *numEntries = stat_infos.size();
+ hdfsFileInfo *file_infos = new hdfsFileInfo[stat_infos.size()];
+ for(std::vector<StatInfo>::size_type i = 0; i < stat_infos.size(); i++) {
+ StatInfoToHdfsFileInfo(&file_infos[i], stat_infos.at(i));
+ }
+
+ return file_infos;
+ } catch (const std::exception & e) {
+ ReportException(e);
*numEntries = 0;
return nullptr;
- }
- if(stat_infos.empty()){
+ } catch (...) {
+ ReportCaughtNonException();
*numEntries = 0;
return nullptr;
}
- *numEntries = stat_infos.size();
- hdfsFileInfo *file_infos = new hdfsFileInfo[stat_infos.size()];
- for(std::vector<StatInfo>::size_type i = 0; i < stat_infos.size(); i++) {
- StatInfoToHdfsFileInfo(&file_infos[i], stat_infos.at(i));
- }
-
- return file_infos;
- } catch (const std::exception & e) {
- ReportException(e);
- *numEntries = 0;
- return nullptr;
- } catch (...) {
- ReportCaughtNonException();
- *numEntries = 0;
- return nullptr;
- }
}
-HDFS_API
void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries)
{
- resetErrorForApiCall("hdfsFreeFileInfo");
- int i;
- for (i = 0; i < numEntries; ++i) {
- delete[] hdfsFileInfo[i].mName;
- delete[] hdfsFileInfo[i].mOwner;
- delete[] hdfsFileInfo[i].mGroup;
- }
- delete[] hdfsFileInfo;
+ errno = 0;
+ int i;
+ for (i = 0; i < numEntries; ++i) {
+ delete[] hdfsFileInfo[i].mName;
+ delete[] hdfsFileInfo[i].mOwner;
+ delete[] hdfsFileInfo[i].mGroup;
+ }
+ delete[] hdfsFileInfo;
}
-HDFS_API
int hdfsCreateDirectory(hdfsFS fs, const char* path) {
- resetErrorForApiCall("hdfsCreateDirectory");
try {
+ errno = 0;
if (!CheckSystem(fs)) {
return -1;
}
@@ -930,39 +891,45 @@ int hdfsCreateDirectory(hdfsFS fs, const char* path) {
return -1;
}
Status stat;
- //Use default permissions and set true for creating all non-existent parent directories
+ //Use default permissions and set true for creating all non-existant parent directories
stat = fs->get_impl()->Mkdirs(*abs_path, FileSystem::GetDefaultPermissionMask(), true);
if (!stat.ok()) {
return Error(stat);
}
return 0;
- } CATCH_AND_REPORT
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
}
-HDFS_API
int hdfsDelete(hdfsFS fs, const char* path, int recursive) {
- resetErrorForApiCall("hdfsDelete");
try {
- if (!CheckSystem(fs)) {
- return -1;
- }
- const optional<std::string> abs_path = getAbsolutePath(fs, path);
- if(!abs_path) {
- return -1;
- }
- Status stat;
- stat = fs->get_impl()->Delete(*abs_path, recursive);
- if (!stat.ok()) {
- return Error(stat);
+ errno = 0;
+ if (!CheckSystem(fs)) {
+ return -1;
+ }
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
+ }
+ Status stat;
+ stat = fs->get_impl()->Delete(*abs_path, recursive);
+ if (!stat.ok()) {
+ return Error(stat);
+ }
+ return 0;
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
}
- return 0;
- } CATCH_AND_REPORT
}
-HDFS_API
int hdfsRename(hdfsFS fs, const char* oldPath, const char* newPath) {
- resetErrorForApiCall("hdfsRename");
try {
+ errno = 0;
if (!CheckSystem(fs)) {
return -1;
}
@@ -977,99 +944,106 @@ int hdfsRename(hdfsFS fs, const char* oldPath, const char* newPath) {
return Error(stat);
}
return 0;
- } CATCH_AND_REPORT
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
}
-HDFS_API
int hdfsChmod(hdfsFS fs, const char* path, short mode){
- resetErrorForApiCall("hdfsChmod");
try {
- if (!CheckSystem(fs)) {
- return -1;
- }
- const optional<std::string> abs_path = getAbsolutePath(fs, path);
- if(!abs_path) {
- return -1;
- }
- Status stat = FileSystem::CheckValidPermissionMask(mode);
- if (!stat.ok()) {
- return Error(stat);
- }
- stat = fs->get_impl()->SetPermission(*abs_path, mode);
- if (!stat.ok()) {
- return Error(stat);
+ errno = 0;
+ if (!CheckSystem(fs)) {
+ return -1;
+ }
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
+ }
+ Status stat = FileSystem::CheckValidPermissionMask(mode);
+ if (!stat.ok()) {
+ return Error(stat);
+ }
+ stat = fs->get_impl()->SetPermission(*abs_path, mode);
+ if (!stat.ok()) {
+ return Error(stat);
+ }
+ return 0;
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
}
- return 0;
- } CATCH_AND_REPORT
}
-HDFS_API
int hdfsChown(hdfsFS fs, const char* path, const char *owner, const char *group){
- resetErrorForApiCall("hdfsChown");
try {
- if (!CheckSystem(fs)) {
- return -1;
- }
- const optional<std::string> abs_path = getAbsolutePath(fs, path);
- if(!abs_path) {
- return -1;
- }
- std::string own = (owner) ? owner : "";
- std::string grp = (group) ? group : "";
+ errno = 0;
+ if (!CheckSystem(fs)) {
+ return -1;
+ }
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return -1;
+ }
+ std::string own = (owner) ? owner : "";
+ std::string grp = (group) ? group : "";
- Status stat;
- stat = fs->get_impl()->SetOwner(*abs_path, own, grp);
- if (!stat.ok()) {
- return Error(stat);
+ Status stat;
+ stat = fs->get_impl()->SetOwner(*abs_path, own, grp);
+ if (!stat.ok()) {
+ return Error(stat);
+ }
+ return 0;
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
}
- return 0;
- } CATCH_AND_REPORT
}
-HDFS_EXT_API
-hdfsFileInfo * hdfsFind(hdfsFS fs, const char* path, const char* name, uint32_t * numEntries)
-{
- resetErrorForApiCall("hdfsFind");
+hdfsFileInfo * hdfsFind(hdfsFS fs, const char* path, const char* name, uint32_t * numEntries){
try {
- if (!CheckSystem(fs)) {
- *numEntries = 0;
- return nullptr;
- }
+ errno = 0;
+ if (!CheckSystem(fs)) {
+ *numEntries = 0;
+ return nullptr;
+ }
- std::vector<StatInfo> stat_infos;
- Status stat = fs->get_impl()->Find(path, name, hdfs::FileSystem::GetDefaultFindMaxDepth(), &stat_infos);
- if (!stat.ok()) {
- Error(stat);
+ std::vector<StatInfo> stat_infos;
+ Status stat = fs->get_impl()->Find(path, name, hdfs::FileSystem::GetDefaultFindMaxDepth(), &stat_infos);
+ if (!stat.ok()) {
+ Error(stat);
+ *numEntries = 0;
+ return nullptr;
+ }
+ //Existing API expects nullptr if size is 0
+ if(stat_infos.empty()){
+ *numEntries = 0;
+ return nullptr;
+ }
+ *numEntries = stat_infos.size();
+ hdfsFileInfo *file_infos = new hdfsFileInfo[stat_infos.size()];
+ for(std::vector<StatInfo>::size_type i = 0; i < stat_infos.size(); i++) {
+ StatInfoToHdfsFileInfo(&file_infos[i], stat_infos.at(i));
+ }
+
+ return file_infos;
+ } catch (const std::exception & e) {
+ ReportException(e);
*numEntries = 0;
return nullptr;
- }
- //Existing API expects nullptr if size is 0
- if(stat_infos.empty()){
+ } catch (...) {
+ ReportCaughtNonException();
*numEntries = 0;
return nullptr;
}
- *numEntries = stat_infos.size();
- hdfsFileInfo *file_infos = new hdfsFileInfo[stat_infos.size()];
- for(std::vector<StatInfo>::size_type i = 0; i < stat_infos.size(); i++) {
- StatInfoToHdfsFileInfo(&file_infos[i], stat_infos.at(i));
- }
-
- return file_infos;
- } catch (const std::exception & e) {
- ReportException(e);
- *numEntries = 0;
- return nullptr;
- } catch (...) {
- ReportCaughtNonException();
- *numEntries = 0;
- return nullptr;
- }
}
-HDFS_EXT_API
int hdfsCreateSnapshot(hdfsFS fs, const char* path, const char* name) {
- resetErrorForApiCall("hdfsCreateSnapshot");
try {
+ errno = 0;
if (!CheckSystem(fs)) {
return -1;
}
@@ -1087,13 +1061,16 @@ int hdfsCreateSnapshot(hdfsFS fs, const char* path, const char* name) {
return Error(stat);
}
return 0;
- } CATCH_AND_REPORT
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
}
-HDFS_EXT_API
int hdfsDeleteSnapshot(hdfsFS fs, const char* path, const char* name) {
- resetErrorForApiCall("hdfsDeleteSnapshot");
try {
+ errno = 0;
if (!CheckSystem(fs)) {
return -1;
}
@@ -1110,13 +1087,16 @@ int hdfsDeleteSnapshot(hdfsFS fs, const char* path, const char* name) {
return Error(stat);
}
return 0;
- } CATCH_AND_REPORT
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
}
-HDFS_EXT_API
int hdfsAllowSnapshot(hdfsFS fs, const char* path) {
- resetErrorForApiCall("hdfsAllowSnapshot");
try {
+ errno = 0;
if (!CheckSystem(fs)) {
return -1;
}
@@ -1130,13 +1110,16 @@ int hdfsAllowSnapshot(hdfsFS fs, const char* path) {
return Error(stat);
}
return 0;
- } CATCH_AND_REPORT
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
}
-HDFS_EXT_API
int hdfsDisallowSnapshot(hdfsFS fs, const char* path) {
- resetErrorForApiCall("hdfsDisallowSnapshot");
try {
+ errno = 0;
if (!CheckSystem(fs)) {
return -1;
}
@@ -1150,15 +1133,18 @@ int hdfsDisallowSnapshot(hdfsFS fs, const char* path) {
return Error(stat);
}
return 0;
- } CATCH_AND_REPORT
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
}
-HDFS_API
tSize hdfsPread(hdfsFS fs, hdfsFile file, tOffset position, void *buffer,
tSize length) {
- resetErrorForApiCall("hdfsPread");
try
{
+ errno = 0;
if (!CheckSystemAndHandle(fs, file)) {
return -1;
}
@@ -1169,14 +1155,17 @@ tSize hdfsPread(hdfsFS fs, hdfsFile file, tOffset position, void *buffer,
return Error(stat);
}
return (tSize)len;
- } CATCH_AND_REPORT
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
}
-HDFS_API
tSize hdfsRead(hdfsFS fs, hdfsFile file, void *buffer, tSize length) {
- resetErrorForApiCall("hdfsRead");
try
{
+ errno = 0;
if (!CheckSystemAndHandle(fs, file)) {
return -1;
}
@@ -1188,63 +1177,67 @@ tSize hdfsRead(hdfsFS fs, hdfsFile file, void *buffer, tSize length) {
}
return (tSize)len;
- } CATCH_AND_REPORT
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
}
-HDFS_API
int hdfsUnbufferFile(hdfsFile file) {
- resetErrorForApiCall("hdfsUnbufferFile");
//Currently we are not doing any buffering
CheckHandle(file);
return -1;
}
-HDFS_API
int hdfsFileGetReadStatistics(hdfsFile file, struct hdfsReadStatistics **stats) {
- resetErrorForApiCall("hdfsFileGetReadStatistics");
try
- {
- if (!CheckHandle(file)) {
- return -1;
+ {
+ errno = 0;
+ if (!CheckHandle(file)) {
+ return -1;
+ }
+ *stats = new hdfsReadStatistics;
+ memset(*stats, 0, sizeof(hdfsReadStatistics));
+ (*stats)->totalBytesRead = file->get_impl()->get_bytes_read();
+ return 0;
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
}
- *stats = new hdfsReadStatistics;
- memset(*stats, 0, sizeof(hdfsReadStatistics));
- (*stats)->totalBytesRead = file->get_impl()->get_bytes_read();
- return 0;
- } CATCH_AND_REPORT
}
-HDFS_API
int hdfsFileClearReadStatistics(hdfsFile file) {
- resetErrorForApiCall("hdfsFileClearReadStatistics");
try
- {
- if (!CheckHandle(file)) {
- return -1;
+ {
+ errno = 0;
+ if (!CheckHandle(file)) {
+ return -1;
+ }
+ file->get_impl()->clear_bytes_read();
+ return 0;
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
}
- file->get_impl()->clear_bytes_read();
- return 0;
- } CATCH_AND_REPORT
}
-HDFS_API
int64_t hdfsReadStatisticsGetRemoteBytesRead(const struct hdfsReadStatistics *stats) {
- resetErrorForApiCall("hdfsReadStatisticsGetRemoteBytesRead");
- return stats->totalBytesRead - stats->totalLocalBytesRead;
+ return stats->totalBytesRead - stats->totalLocalBytesRead;
}
-HDFS_API
void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats) {
- resetErrorForApiCall("hdfsFileFreeReadStatistics");
- delete stats;
+ errno = 0;
+ delete stats;
}
/* 0 on success, -1 on error*/
-HDFS_API
int hdfsSeek(hdfsFS fs, hdfsFile file, tOffset desiredPos) {
- resetErrorForApiCall("hdfsSeek");
try
{
+ errno = 0;
if (!CheckSystemAndHandle(fs, file)) {
return -1;
}
@@ -1256,14 +1249,17 @@ int hdfsSeek(hdfsFS fs, hdfsFile file, tOffset desiredPos) {
}
return 0;
- } CATCH_AND_REPORT
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
}
-HDFS_API
tOffset hdfsTell(hdfsFS fs, hdfsFile file) {
- resetErrorForApiCall("hdfsTell");
try
{
+ errno = 0;
if (!CheckSystemAndHandle(fs, file)) {
return -1;
}
@@ -1275,30 +1271,36 @@ tOffset hdfsTell(hdfsFS fs, hdfsFile file) {
}
return offset;
- } CATCH_AND_REPORT
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
}
/* extended API */
-HDFS_EXT_API
int hdfsCancel(hdfsFS fs, hdfsFile file) {
- resetErrorForApiCall("hdfsCancel");
try
{
+ errno = 0;
if (!CheckSystemAndHandle(fs, file)) {
return -1;
}
static_cast<FileHandleImpl*>(file->get_impl())->CancelOperations();
return 0;
- } CATCH_AND_REPORT
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
}
-HDFS_EXT_API
int hdfsGetBlockLocations(hdfsFS fs, const char *path, struct hdfsBlockLocations ** locations_out)
{
- resetErrorForApiCall("hdfsGetBlockLocations");
try
{
+ errno = 0;
if (!CheckSystem(fs)) {
return -1;
}
@@ -1362,12 +1364,15 @@ int hdfsGetBlockLocations(hdfsFS fs, const char *path, struct hdfsBlockLocations
}
return 0;
- } CATCH_AND_REPORT
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
}
-HDFS_EXT_API
int hdfsFreeBlockLocations(struct hdfsBlockLocations * blockLocations) {
- resetErrorForApiCall("hdfsFreeBlockLocations");
+ errno = 0;
if (blockLocations == nullptr)
return 0;
@@ -1386,50 +1391,48 @@ int hdfsFreeBlockLocations(struct hdfsBlockLocations * blockLocations) {
return 0;
}
-HDFS_API
char*** hdfsGetHosts(hdfsFS fs, const char* path, tOffset start, tOffset length) {
- resetErrorForApiCall("hdfsGetHosts");
try
- {
- if (!CheckSystem(fs)) {
- return nullptr;
- }
- const optional<std::string> abs_path = getAbsolutePath(fs, path);
- if(!abs_path) {
+ {
+ errno = 0;
+ if (!CheckSystem(fs)) {
+ return nullptr;
+ }
+ const optional<std::string> abs_path = getAbsolutePath(fs, path);
+ if(!abs_path) {
+ return nullptr;
+ }
+ std::shared_ptr<FileBlockLocation> ppLocations;
+ Status stat = fs->get_impl()->GetBlockLocations(*abs_path, start, length, &ppLocations);
+ if (!stat.ok()) {
+ Error(stat);
+ return nullptr;
+ }
+ const std::vector<BlockLocation> & ppBlockLocations = ppLocations->getBlockLocations();
+ char ***hosts = new char**[ppBlockLocations.size() + 1];
+ for (size_t i=0; i < ppBlockLocations.size(); i++) {
+ const std::vector<DNInfo> & ppDNInfos = ppBlockLocations[i].getDataNodes();
+ hosts[i] = new char*[ppDNInfos.size() + 1];
+ for (size_t j=0; j < ppDNInfos.size(); j++) {
+ auto ppDNInfo = ppDNInfos[j];
+ hosts[i][j] = new char[ppDNInfo.getHostname().size() + 1];
+ strncpy(hosts[i][j], ppDNInfo.getHostname().c_str(), ppDNInfo.getHostname().size() + 1);
+ }
+ hosts[i][ppDNInfos.size()] = nullptr;
+ }
+ hosts[ppBlockLocations.size()] = nullptr;
+ return hosts;
+ } catch (const std::exception & e) {
+ ReportException(e);
return nullptr;
- }
- std::shared_ptr<FileBlockLocation> ppLocations;
- Status stat = fs->get_impl()->GetBlockLocations(*abs_path, start, length, &ppLocations);
- if (!stat.ok()) {
- Error(stat);
+ } catch (...) {
+ ReportCaughtNonException();
return nullptr;
}
- const std::vector<BlockLocation> & ppBlockLocations = ppLocations->getBlockLocations();
- char ***hosts = new char**[ppBlockLocations.size() + 1];
- for (size_t i=0; i < ppBlockLocations.size(); i++) {
- const std::vector<DNInfo> & ppDNInfos = ppBlockLocations[i].getDataNodes();
- hosts[i] = new char*[ppDNInfos.size() + 1];
- for (size_t j=0; j < ppDNInfos.size(); j++) {
- auto ppDNInfo = ppDNInfos[j];
- hosts[i][j] = new char[ppDNInfo.getHostname().size() + 1];
- strncpy(hosts[i][j], ppDNInfo.getHostname().c_str(), ppDNInfo.getHostname().size() + 1);
- }
- hosts[i][ppDNInfos.size()] = nullptr;
- }
- hosts[ppBlockLocations.size()] = nullptr;
- return hosts;
- } catch (const std::exception & e) {
- ReportException(e);
- return nullptr;
- } catch (...) {
- ReportCaughtNonException();
- return nullptr;
- }
}
-HDFS_API
void hdfsFreeHosts(char ***blockHosts) {
- resetErrorForApiCall("hdfsFreeHosts");
+ errno = 0;
if (blockHosts == nullptr)
return;
@@ -1492,20 +1495,16 @@ event_response file_callback_glue(libhdfspp_file_event_callback handler,
return event_response::make_ok();
}
-HDFS_EXT_API
int hdfsPreAttachFSMonitor(libhdfspp_fs_event_callback handler, int64_t cookie)
{
- resetErrorForApiCall("hdfsPreAttachFSMonitor");
fs_event_callback callback = std::bind(fs_callback_glue, handler, cookie, _1, _2, _3);
fsEventCallback = callback;
return 0;
}
-HDFS_EXT_API
int hdfsPreAttachFileMonitor(libhdfspp_file_event_callback handler, int64_t cookie)
{
- resetErrorForApiCall("hdfsPreAttachFileMonitor");
file_event_callback callback = std::bind(file_callback_glue, handler, cookie, _1, _2, _3, _4);
fileEventCallback = callback;
return 0;
@@ -1542,12 +1541,11 @@ hdfsBuilder::hdfsBuilder(const char * directory) :
config = LoadDefault(loader);
}
-HDFS_API
struct hdfsBuilder *hdfsNewBuilder(void)
{
- resetErrorForApiCall("hdfsNewBuilder");
try
{
+ errno = 0;
return new struct hdfsBuilder();
} catch (const std::exception & e) {
ReportException(e);
@@ -1558,42 +1556,37 @@ struct hdfsBuilder *hdfsNewBuilder(void)
}
}
-HDFS_API
void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn)
{
- resetErrorForApiCall("hdfsBuilderSetNameNode");
+ errno = 0;
bld->overrideHost = std::string(nn);
}
-HDFS_API
void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port)
{
- resetErrorForApiCall("hdfsBuilderSetNameNodePort");
+ errno = 0;
bld->overridePort = port;
}
-HDFS_API
void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName)
{
- resetErrorForApiCall("hdfsBuilderSetUserName");
+ errno = 0;
if (userName && *userName) {
bld->user = std::string(userName);
}
}
-HDFS_API
void hdfsBuilderSetForceNewInstance(struct hdfsBuilder *bld) {
- resetErrorForApiCall("hdfsBuilderSetForceNewInstance");
//libhdfspp always returns a new instance, so nothing to do
(void)bld;
+ errno = 0;
}
-HDFS_API
void hdfsFreeBuilder(struct hdfsBuilder *bld)
{
- resetErrorForApiCall("hdfsFreeBuilder");
try
{
+ errno = 0;
delete bld;
} catch (const std::exception & e) {
ReportException(e);
@@ -1602,12 +1595,12 @@ void hdfsFreeBuilder(struct hdfsBuilder *bld)
}
}
-HDFS_API
-int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key, const char *val)
+int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key,
+ const char *val)
{
- resetErrorForApiCall("hdfsBuilderConfSetStr");
try
{
+ errno = 0;
optional<HdfsConfiguration> newConfig = bld->loader.OverlayValue(bld->config, key, val);
if (newConfig)
{
@@ -1619,17 +1612,6 @@ int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key, const char *
ReportError(EINVAL, "Could not change Builder value");
return -1;
}
- } CATCH_AND_REPORT
-}
-
-HDFS_API
-int hdfsConfGetStr(const char *key, char **val)
-{
- resetErrorForApiCall("hdfsConfGetStr");
- try
- {
- hdfsBuilder builder;
- return hdfsBuilderConfGetStr(&builder, key, val);
} catch (const std::exception & e) {
return ReportException(e);
} catch (...) {
@@ -1637,39 +1619,52 @@ int hdfsConfGetStr(const char *key, char **val)
}
}
-HDFS_API
void hdfsConfStrFree(char *val)
{
- resetErrorForApiCall("hdfsConfStrFree");
+ errno = 0;
free(val);
}
-HDFS_API
hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) {
- resetErrorForApiCall("hdfsBuilderConnect");
return doHdfsConnect(bld->overrideHost, bld->overridePort, bld->user, bld->config.GetOptions());
}
-HDFS_API
+int hdfsConfGetStr(const char *key, char **val)
+{
+ try
+ {
+ errno = 0;
+ hdfsBuilder builder;
+ return hdfsBuilderConfGetStr(&builder, key, val);
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
+}
+
int hdfsConfGetInt(const char *key, int32_t *val)
{
- resetErrorForApiCall("hdfsConfGetInt");
try
{
+ errno = 0;
hdfsBuilder builder;
return hdfsBuilderConfGetInt(&builder, key, val);
- } CATCH_AND_REPORT
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
}
//
// Extended builder interface
//
-HDFS_EXT_API
struct hdfsBuilder *hdfsNewBuilderFromDirectory(const char * configDirectory)
{
- resetErrorForApiCall("hdfsNewBuilderFromDirectory");
try
{
+ errno = 0;
return new struct hdfsBuilder(configDirectory);
} catch (const std::exception & e) {
ReportException(e);
@@ -1680,12 +1675,12 @@ struct hdfsBuilder *hdfsNewBuilderFromDirectory(const char * configDirectory)
}
}
-HDFS_EXT_API
-int hdfsBuilderConfGetStr(struct hdfsBuilder *bld, const char *key, char **val)
+int hdfsBuilderConfGetStr(struct hdfsBuilder *bld, const char *key,
+ char **val)
{
- resetErrorForApiCall("hdfsBuilderConfGetStr");
try
{
+ errno = 0;
optional<std::string> value = bld->config.Get(key);
if (value)
{
@@ -1698,7 +1693,11 @@ int hdfsBuilderConfGetStr(struct hdfsBuilder *bld, const char *key, char **val)
*val = nullptr;
}
return 0;
- } CATCH_AND_REPORT
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
}
// If we're running on a 32-bit platform, we might get 64-bit values that
@@ -1709,12 +1708,11 @@ bool isValidInt(int64_t value)
value <= std::numeric_limits<int>::max());
}
-HDFS_EXT_API
int hdfsBuilderConfGetInt(struct hdfsBuilder *bld, const char *key, int32_t *val)
{
- resetErrorForApiCall("hdfsBuilderConfGetInt");
try
{
+ errno = 0;
// Pull from default configuration
optional<int64_t> value = bld->config.GetInt(key);
if (value)
@@ -1729,15 +1727,18 @@ int hdfsBuilderConfGetInt(struct hdfsBuilder *bld, const char *key, int32_t *val
// If not found, don't change val
ReportError(EINVAL, "Could not get Builder value");
return 0;
- } CATCH_AND_REPORT
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
}
-HDFS_EXT_API
int hdfsBuilderConfGetLong(struct hdfsBuilder *bld, const char *key, int64_t *val)
{
- resetErrorForApiCall("hdfsBuilderConfGetLong");
try
{
+ errno = 0;
// Pull from default configuration
optional<int64_t> value = bld->config.GetInt(key);
if (value)
@@ -1748,7 +1749,11 @@ int hdfsBuilderConfGetLong(struct hdfsBuilder *bld, const char *key, int64_t *va
// If not found, don't change val
ReportError(EINVAL, "Could not get Builder value");
return 0;
- } CATCH_AND_REPORT
+ } catch (const std::exception & e) {
+ return ReportException(e);
+ } catch (...) {
+ return ReportCaughtNonException();
+ }
}
/**
@@ -1824,21 +1829,15 @@ void CForwardingLogger::FreeLogData(LogData *data) {
}
-HDFS_EXT_API
LogData *hdfsCopyLogData(LogData *data) {
- resetErrorForApiCall("hdfsCopyLogData");
return CForwardingLogger::CopyLogData(data);
}
-HDFS_EXT_API
void hdfsFreeLogData(LogData *data) {
- resetErrorForApiCall("hdfsFreeLogData");
CForwardingLogger::FreeLogData(data);
}
-HDFS_EXT_API
void hdfsSetLogFunction(void (*callback)(LogData*)) {
- resetErrorForApiCall("hdfsSetLogFunction");
CForwardingLogger *logger = new CForwardingLogger();
logger->SetCallback(callback);
LogManager::SetLoggerImplementation(std::unique_ptr<LoggerInterface>(logger));
@@ -1850,8 +1849,8 @@ static bool IsLevelValid(int component) {
return true;
}
+
// should use __builtin_popcnt as optimization on some platforms
-// *but maybe not Intel: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=62011
static int popcnt(int val) {
int bits = sizeof(val) * 8;
int count = 0;
@@ -1870,27 +1869,24 @@ static bool IsComponentValid(int component) {
return true;
}
-HDFS_EXT_API
int hdfsEnableLoggingForComponent(int component) {
- resetErrorForApiCall("hdfsEnableLoggingForComponent");
+ errno = 0;
if(!IsComponentValid(component))
return -1;
LogManager::EnableLogForComponent(static_cast<LogSourceComponent>(component));
return 0;
}
-HDFS_EXT_API
int hdfsDisableLoggingForComponent(int component) {
- resetErrorForApiCall("hdfsDisableLoggingForComponent");
+ errno = 0;
if(!IsComponentValid(component))
return -1;
LogManager::DisableLogForComponent(static_cast<LogSourceComponent>(component));
return 0;
}
-HDFS_EXT_API
int hdfsSetLoggingLevel(int level) {
- resetErrorForApiCall("hdfsSetLoggingLevel");
+ errno = 0;
if(!IsLevelValid(level))
return -1;
LogManager::SetLogLevel(static_cast<LogLevel>(level));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/52] [abbrv] hadoop git commit: HDFS-11908: libhdfs++:
Authentication failure when first NN of a kerberized HA cluster is standby.
Contributed by James Clampffer
Posted by jh...@apache.org.
HDFS-11908: libhdfs++: Authentication failure when first NN of a kerberized HA cluster is standby. Contributed by James Clampffer
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23afd508
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23afd508
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23afd508
Branch: refs/heads/HDFS-8707
Commit: 23afd508651c5e4a10cf2c4784b543a6e77f3791
Parents: e64ba87
Author: James Clampffer <ja...@hp.com>
Authored: Mon Jul 10 13:47:51 2017 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../src/main/native/libhdfspp/lib/rpc/rpc_connection.h | 1 +
.../main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc | 5 +++++
.../src/main/native/libhdfspp/lib/rpc/rpc_engine.cc | 10 ++++++----
3 files changed, 12 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23afd508/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
index 7a671fe..449a1b6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
@@ -80,6 +80,7 @@ class RpcConnection : public std::enable_shared_from_this<RpcConnection> {
void SetEventHandlers(std::shared_ptr<LibhdfsEvents> event_handlers);
void SetClusterName(std::string cluster_name);
+ void SetAuthInfo(const AuthInfo& auth_info);
LockFreeRpcEngine *engine() { return engine_; }
::asio::io_service &io_service();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23afd508/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
index 198bcdc..06b1616 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
@@ -383,6 +383,11 @@ void RpcConnection::SetClusterName(std::string cluster_name) {
cluster_name_ = cluster_name;
}
+void RpcConnection::SetAuthInfo(const AuthInfo& auth_info) {
+ std::lock_guard<std::mutex> state_lock(connection_state_lock_);
+ auth_info_ = auth_info;
+}
+
void RpcConnection::CommsError(const Status &status) {
assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
LOG_DEBUG(kRPC, << "RpcConnection::CommsError called");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23afd508/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
index 22c0e74..98c41da 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
@@ -176,10 +176,12 @@ std::shared_ptr<RpcConnection> RpcEngine::NewConnection()
std::shared_ptr<RpcConnection> RpcEngine::InitializeConnection()
{
- std::shared_ptr<RpcConnection> result = NewConnection();
- result->SetEventHandlers(event_handlers_);
- result->SetClusterName(cluster_name_);
- return result;
+ std::shared_ptr<RpcConnection> newConn = NewConnection();
+ newConn->SetEventHandlers(event_handlers_);
+ newConn->SetClusterName(cluster_name_);
+ newConn->SetAuthInfo(auth_info_);
+
+ return newConn;
}
void RpcEngine::AsyncRpcCommsError(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/52] [abbrv] hadoop git commit: HDFS-10685: libhdfs++: return
explicit error when non-secured client connects to secured server.
Contributed by Kai Jiang.
Posted by jh...@apache.org.
HDFS-10685: libhdfs++: return explicit error when non-secured client connects to secured server. Contributed by Kai Jiang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/58de2df8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/58de2df8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/58de2df8
Branch: refs/heads/HDFS-8707
Commit: 58de2df8606cd598b41785c0c5c6bf98d66ebc15
Parents: 69d5534
Author: James <jh...@apache.org>
Authored: Wed Jan 11 10:54:50 2017 -0500
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../src/main/native/libhdfspp/include/hdfspp/status.h | 1 +
.../src/main/native/libhdfspp/lib/common/status.cc | 11 ++++++++++-
.../main/native/libhdfspp/lib/rpc/cyrus_sasl_engine.cc | 2 +-
.../src/main/native/libhdfspp/lib/rpc/rpc_connection.cc | 6 +++++-
4 files changed, 17 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58de2df8/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
index 39cf8c6..d2c32b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
@@ -43,6 +43,7 @@ class Status {
static Status Exception(const char *exception_class_name, const char *exception_details);
static Status Error(const char *error_message);
static Status AuthenticationFailed();
+ static Status AuthenticationFailed(const char *msg);
static Status Canceled();
static Status PathNotFound(const char *msg);
static Status InvalidOffset(const char *msg);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58de2df8/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
index 15d304a..590a036 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
@@ -131,7 +131,16 @@ Status Status::Error(const char *error_message) {
}
Status Status::AuthenticationFailed() {
- return Status(kAuthenticationFailed, "Authentication failed");
+ return Status::AuthenticationFailed(nullptr);
+}
+
+Status Status::AuthenticationFailed(const char *msg) {
+ std::string formatted = "AuthenticationFailed";
+ if(msg) {
+ formatted += ": ";
+ formatted += msg;
+ }
+ return Status(kAuthenticationFailed, formatted.c_str());
}
Status Status::Canceled() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58de2df8/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/cyrus_sasl_engine.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/cyrus_sasl_engine.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/cyrus_sasl_engine.cc
index ca82305..69b2267 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/cyrus_sasl_engine.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/cyrus_sasl_engine.cc
@@ -91,7 +91,7 @@ Status make_status(int rc) {
if (rc != SASL_OK &&
rc != SASL_CONTINUE &&
rc != SASL_INTERACT) {
- return Status::Error(errStr(rc).c_str());
+ return Status::AuthenticationFailed(errStr(rc).c_str());
}
return Status::OK();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58de2df8/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
index 2068614..f629d1f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
@@ -293,7 +293,11 @@ Status RpcConnection::HandleRpcResponse(std::shared_ptr<Response> response) {
auto req = RemoveFromRunningQueue(h.callid());
if (!req) {
LOG_WARN(kRPC, << "RPC response with Unknown call id " << h.callid());
- return Status::Error("Rpc response with unknown call id");
+ if((int32_t)h.callid() == RpcEngine::kCallIdSasl) {
+ return Status::AuthenticationFailed("You have an unsecured client connecting to a secured server");
+ } else {
+ return Status::Error("Rpc response with unknown call id");
+ }
}
Status status;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[51/52] [abbrv] hadoop git commit: HDFS-12427: libhdfs++: Prevent
requests from holding dangling pointer to RpcEngine. Contributed by James
Clampffer.
Posted by jh...@apache.org.
HDFS-12427: libhdfs++: Prevent requests from holding dangling pointer to RpcEngine. Contributed by James Clampffer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fc5e44d9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fc5e44d9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fc5e44d9
Branch: refs/heads/HDFS-8707
Commit: fc5e44d9edeaad9c1381a46fb370b167ac6250b1
Parents: 48db24a
Author: James <jh...@apache.org>
Authored: Mon Oct 30 13:54:05 2017 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../main/native/libhdfspp/lib/fs/filesystem.h | 2 +
.../libhdfspp/lib/fs/namenode_operations.cc | 6 +-
.../libhdfspp/lib/fs/namenode_operations.h | 14 +-
.../libhdfspp/lib/proto/protoc_gen_hrpc.cc | 4 +-
.../main/native/libhdfspp/lib/rpc/request.cc | 61 ++++-----
.../src/main/native/libhdfspp/lib/rpc/request.h | 10 +-
.../native/libhdfspp/lib/rpc/rpc_connection.h | 17 +--
.../libhdfspp/lib/rpc/rpc_connection_impl.cc | 131 +++++++++++++------
.../libhdfspp/lib/rpc/rpc_connection_impl.h | 54 +++++---
.../main/native/libhdfspp/lib/rpc/rpc_engine.cc | 2 +-
.../main/native/libhdfspp/lib/rpc/rpc_engine.h | 30 ++---
.../native/libhdfspp/tests/rpc_engine_test.cc | 69 +++++-----
12 files changed, 240 insertions(+), 160 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc5e44d9/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
index ac6c3d8..f2e9abd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
@@ -212,6 +212,8 @@ private:
* The IoService must be the first member variable to ensure that it gets
* destroyed last. This allows other members to dequeue things from the
* service in their own destructors.
+ * A side effect of this is that requests may outlive the RpcEngine they
+ * reference.
**/
std::shared_ptr<IoServiceImpl> io_service_;
const Options options_;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc5e44d9/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
index b5a6564..e46faad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
@@ -42,11 +42,11 @@ namespace hdfs {
void NameNodeOperations::Connect(const std::string &cluster_name,
const std::vector<ResolvedNamenodeInfo> &servers,
std::function<void(const Status &)> &&handler) {
- engine_.Connect(cluster_name, servers, handler);
+ engine_->Connect(cluster_name, servers, handler);
}
bool NameNodeOperations::CancelPendingConnect() {
- return engine_.CancelPendingConnect();
+ return engine_->CancelPendingConnect();
}
void NameNodeOperations::GetBlockLocations(const std::string & path, uint64_t offset, uint64_t length,
@@ -678,7 +678,7 @@ void NameNodeOperations::DisallowSnapshot(const std::string & path, std::functio
}
void NameNodeOperations::SetFsEventCallback(fs_event_callback callback) {
- engine_.SetFsEventCallback(callback);
+ engine_->SetFsEventCallback(callback);
}
void NameNodeOperations::HdfsFileStatusProtoToStatInfo(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc5e44d9/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
index 97c0e18..f4caa18 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
@@ -46,8 +46,9 @@ public:
const std::string &client_name, const std::string &user_name,
const char *protocol_name, int protocol_version) :
io_service_(io_service),
- engine_(io_service, options, client_name, user_name, protocol_name, protocol_version),
- namenode_(& engine_), options_(options) {}
+ engine_(std::make_shared<RpcEngine>(io_service, options, client_name, user_name, protocol_name, protocol_version)),
+ namenode_(engine_), options_(options) {}
+
void Connect(const std::string &cluster_name,
const std::vector<ResolvedNamenodeInfo> &servers,
@@ -119,7 +120,14 @@ private:
static void GetFsStatsResponseProtoToFsInfo(hdfs::FsInfo & fs_info, const std::shared_ptr<::hadoop::hdfs::GetFsStatsResponseProto> & fs);
::asio::io_service * io_service_;
- RpcEngine engine_;
+
+ // This is the only permanent owner of the RpcEngine, however the RPC layer
+ // needs to reference count it prevent races during FileSystem destruction.
+ // In order to do this they hold weak_ptrs and promote them to shared_ptr
+ // when calling non-blocking RpcEngine methods e.g. get_client_id().
+ std::shared_ptr<RpcEngine> engine_;
+
+ // Automatically generated methods for RPC calls. See protoc_gen_hrpc.cc
ClientNamenodeProtocol namenode_;
const Options options_;
};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc5e44d9/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/protoc_gen_hrpc.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/protoc_gen_hrpc.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/protoc_gen_hrpc.cc
index bb0d8d8..e7355c0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/protoc_gen_hrpc.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/protoc_gen_hrpc.cc
@@ -64,11 +64,11 @@ void StubGenerator::EmitService(const ServiceDescriptor *service,
out->Print("\n// GENERATED AUTOMATICALLY. DO NOT MODIFY.\n"
"class $service$ {\n"
"private:\n"
- " ::hdfs::RpcEngine *const engine_;\n"
+ " std::shared_ptr<::hdfs::RpcEngine> engine_;\n"
"public:\n"
" typedef std::function<void(const ::hdfs::Status &)> Callback;\n"
" typedef ::google::protobuf::MessageLite Message;\n"
- " inline $service$(::hdfs::RpcEngine *engine)\n"
+ " inline $service$(std::shared_ptr<::hdfs::RpcEngine> engine)\n"
" : engine_(engine) {}\n",
"service", service->name());
for (int i = 0; i < service->method_count(); ++i) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc5e44d9/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc
index 119962a..356411e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc
@@ -38,6 +38,11 @@ using namespace ::std::placeholders;
static const int kNoRetry = -1;
// Protobuf helper functions.
+// Note/todo: Using the zero-copy protobuf API here makes the simple procedures
+// below tricky to read and debug while providing minimal benefit. Reducing
+// allocations in BlockReader (HDFS-11266) and smarter use of std::stringstream
+// will have a much larger impact according to cachegrind profiles on common
+// workloads.
static void AddHeadersToPacket(std::string *res,
std::initializer_list<const pb::MessageLite *> headers,
const std::string *payload) {
@@ -82,50 +87,33 @@ static void ConstructPayload(std::string *res, const pb::MessageLite *header) {
buf = header->SerializeWithCachedSizesToArray(buf);
}
-static void ConstructPayload(std::string *res, const std::string *request) {
- int len =
- pbio::CodedOutputStream::VarintSize32(request->size()) + request->size();
- res->reserve(len);
- pbio::StringOutputStream ss(res);
- pbio::CodedOutputStream os(&ss);
- uint8_t *buf = os.GetDirectBufferForNBytesAndAdvance(len);
- assert(buf);
- buf = pbio::CodedOutputStream::WriteVarint32ToArray(request->size(), buf);
- buf = os.WriteStringToArray(*request, buf);
-}
-
-static void SetRequestHeader(LockFreeRpcEngine *engine, int call_id,
+static void SetRequestHeader(std::weak_ptr<LockFreeRpcEngine> weak_engine, int call_id,
const std::string &method_name, int retry_count,
RpcRequestHeaderProto *rpc_header,
- RequestHeaderProto *req_header) {
+ RequestHeaderProto *req_header)
+{
+ // Ensure the RpcEngine is live. If it's not then the FileSystem is being destructed.
+ std::shared_ptr<LockFreeRpcEngine> counted_engine = weak_engine.lock();
+ if(!counted_engine) {
+ LOG_ERROR(kRPC, << "SetRequestHeader attempted to access an invalid RpcEngine");
+ return;
+ }
+
rpc_header->set_rpckind(RPC_PROTOCOL_BUFFER);
rpc_header->set_rpcop(RpcRequestHeaderProto::RPC_FINAL_PACKET);
rpc_header->set_callid(call_id);
- if (retry_count != kNoRetry)
+ if (retry_count != kNoRetry) {
rpc_header->set_retrycount(retry_count);
- rpc_header->set_clientid(engine->client_id());
-
+ }
+ rpc_header->set_clientid(counted_engine->client_id());
req_header->set_methodname(method_name);
- req_header->set_declaringclassprotocolname(engine->protocol_name());
- req_header->set_clientprotocolversion(engine->protocol_version());
+ req_header->set_declaringclassprotocolname(counted_engine->protocol_name());
+ req_header->set_clientprotocolversion(counted_engine->protocol_version());
}
// Request implementation
-Request::Request(LockFreeRpcEngine *engine, const std::string &method_name, int call_id,
- const std::string &request, Handler &&handler)
- : engine_(engine),
- method_name_(method_name),
- call_id_(call_id),
- timer_(engine->io_service()),
- handler_(std::move(handler)),
- retry_count_(engine->retry_policy() ? 0 : kNoRetry),
- failover_count_(0) {
- ConstructPayload(&payload_, &request);
-}
-
-
-Request::Request(LockFreeRpcEngine *engine, const std::string &method_name, int call_id,
+Request::Request(std::shared_ptr<LockFreeRpcEngine> engine, const std::string &method_name, int call_id,
const pb::MessageLite *request, Handler &&handler)
: engine_(engine),
method_name_(method_name),
@@ -133,13 +121,14 @@ Request::Request(LockFreeRpcEngine *engine, const std::string &method_name, int
timer_(engine->io_service()),
handler_(std::move(handler)),
retry_count_(engine->retry_policy() ? 0 : kNoRetry),
- failover_count_(0) {
+ failover_count_(0)
+{
ConstructPayload(&payload_, request);
}
-Request::Request(LockFreeRpcEngine *engine, Handler &&handler)
+Request::Request(std::shared_ptr<LockFreeRpcEngine> engine, Handler &&handler)
: engine_(engine),
- call_id_(-1),
+ call_id_(-1/*Handshake ID*/),
timer_(engine->io_service()),
handler_(std::move(handler)),
retry_count_(engine->retry_policy() ? 0 : kNoRetry),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc5e44d9/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.h
index d265a4c..f195540 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.h
@@ -23,6 +23,7 @@
#include "common/new_delete.h"
#include <string>
+#include <memory>
#include <google/protobuf/message_lite.h>
#include <google/protobuf/io/coded_stream.h>
@@ -48,14 +49,13 @@ class Request {
typedef std::function<void(::google::protobuf::io::CodedInputStream *is,
const Status &status)> Handler;
- Request(LockFreeRpcEngine *engine, const std::string &method_name, int call_id,
- const std::string &request, Handler &&callback);
- Request(LockFreeRpcEngine *engine, const std::string &method_name, int call_id,
+ // Constructors will not make any blocking calls while holding the shared_ptr<RpcEngine>
+ Request(std::shared_ptr<LockFreeRpcEngine> engine, const std::string &method_name, int call_id,
const ::google::protobuf::MessageLite *request, Handler &&callback);
// Null request (with no actual message) used to track the state of an
// initial Connect call
- Request(LockFreeRpcEngine *engine, Handler &&handler);
+ Request(std::shared_ptr<LockFreeRpcEngine> engine, Handler &&handler);
int call_id() const { return call_id_; }
std::string method_name() const { return method_name_; }
@@ -71,7 +71,7 @@ class Request {
std::string GetDebugString() const;
private:
- LockFreeRpcEngine *const engine_;
+ std::weak_ptr<LockFreeRpcEngine> engine_;
const std::string method_name_;
const int call_id_;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc5e44d9/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
index 449a1b6..9e54983 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
@@ -52,7 +52,7 @@ class SaslProtocol;
class RpcConnection : public std::enable_shared_from_this<RpcConnection> {
public:
MEMCHECKED_CLASS(RpcConnection)
- RpcConnection(LockFreeRpcEngine *engine);
+ RpcConnection(std::shared_ptr<LockFreeRpcEngine> engine);
virtual ~RpcConnection();
// Note that a single server can have multiple endpoints - especially both
@@ -82,8 +82,8 @@ class RpcConnection : public std::enable_shared_from_this<RpcConnection> {
void SetClusterName(std::string cluster_name);
void SetAuthInfo(const AuthInfo& auth_info);
- LockFreeRpcEngine *engine() { return engine_; }
- ::asio::io_service &io_service();
+ std::weak_ptr<LockFreeRpcEngine> engine() { return engine_; }
+ ::asio::io_service *GetIoService();
protected:
struct Response {
@@ -139,7 +139,7 @@ class RpcConnection : public std::enable_shared_from_this<RpcConnection> {
void ClearAndDisconnect(const ::asio::error_code &ec);
std::shared_ptr<Request> RemoveFromRunningQueue(int call_id);
- LockFreeRpcEngine *const engine_;
+ std::weak_ptr<LockFreeRpcEngine> engine_;
std::shared_ptr<Response> current_response_state_;
AuthInfo auth_info_;
@@ -158,16 +158,17 @@ class RpcConnection : public std::enable_shared_from_this<RpcConnection> {
// State machine for performing a SASL handshake
std::shared_ptr<SaslProtocol> sasl_protocol_;
- // The request being sent over the wire; will also be in requests_on_fly_
- std::shared_ptr<Request> request_over_the_wire_;
+ // The request being sent over the wire; will also be in sent_requests_
+ std::shared_ptr<Request> outgoing_request_;
// Requests to be sent over the wire
std::deque<std::shared_ptr<Request>> pending_requests_;
// Requests to be sent over the wire during authentication; not retried if
// there is a connection error
std::deque<std::shared_ptr<Request>> auth_requests_;
// Requests that are waiting for responses
- typedef std::unordered_map<int, std::shared_ptr<Request>> RequestOnFlyMap;
- RequestOnFlyMap requests_on_fly_;
+ typedef std::unordered_map<int, std::shared_ptr<Request>> SentRequestMap;
+ SentRequestMap sent_requests_;
+
std::shared_ptr<LibhdfsEvents> event_handlers_;
std::string cluster_name_;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc5e44d9/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
index 947d121..ee9b704 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
@@ -66,17 +66,29 @@ static void AddHeadersToPacket(
RpcConnection::~RpcConnection() {}
-RpcConnection::RpcConnection(LockFreeRpcEngine *engine)
+RpcConnection::RpcConnection(std::shared_ptr<LockFreeRpcEngine> engine)
: engine_(engine),
connected_(kNotYetConnected) {}
-::asio::io_service &RpcConnection::io_service() {
- return engine_->io_service();
+::asio::io_service *RpcConnection::GetIoService() {
+ std::shared_ptr<LockFreeRpcEngine> pinnedEngine = engine_.lock();
+ if(!pinnedEngine) {
+ LOG_ERROR(kRPC, << "RpcConnection@" << this << " attempted to access invalid RpcEngine");
+ return nullptr;
+ }
+
+ return &pinnedEngine->io_service();
}
void RpcConnection::StartReading() {
auto shared_this = shared_from_this();
- io_service().post([shared_this, this] () {
+ ::asio::io_service *service = GetIoService();
+ if(!service) {
+ LOG_ERROR(kRPC, << "RpcConnection@" << this << " attempted to access invalid IoService");
+ return;
+ }
+
+ service->post([shared_this, this] () {
OnRecvCompleted(::asio::error_code(), 0);
});
}
@@ -151,12 +163,19 @@ void RpcConnection::ContextComplete(const Status &s) {
void RpcConnection::AsyncFlushPendingRequests() {
std::shared_ptr<RpcConnection> shared_this = shared_from_this();
- io_service().post([shared_this, this]() {
+
+ ::asio::io_service *service = GetIoService();
+ if(!service) {
+ LOG_ERROR(kRPC, << "RpcConnection@" << this << " attempted to access invalid IoService");
+ return;
+ }
+
+ service->post([shared_this, this]() {
std::lock_guard<std::mutex> state_lock(connection_state_lock_);
LOG_TRACE(kRPC, << "RpcConnection::AsyncFlushPendingRequests called (connected=" << ToString(connected_) << ")");
- if (!request_over_the_wire_) {
+ if (!outgoing_request_) {
FlushPendingRequests();
}
});
@@ -209,7 +228,13 @@ Status RpcConnection::HandleRpcResponse(std::shared_ptr<Response> response) {
return status;
}
- io_service().post([req, response, status]() {
+ ::asio::io_service *service = GetIoService();
+ if(!service) {
+ LOG_ERROR(kRPC, << "RpcConnection@" << this << " attempted to access invalid IoService");
+ return Status::Error("RpcConnection attempted to access invalid IoService");
+ }
+
+ service->post([req, response, status]() {
req->OnResponseArrived(response->in.get(), status); // Never call back while holding a lock
});
@@ -267,23 +292,29 @@ std::shared_ptr<std::string> RpcConnection::PrepareContextPacket() {
// after the SASL handshake (if any)
assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
- auto res = std::make_shared<std::string>();
+ std::shared_ptr<LockFreeRpcEngine> pinnedEngine = engine_.lock();
+ if(!pinnedEngine) {
+ LOG_ERROR(kRPC, << "RpcConnection@" << this << " attempted to access invalid RpcEngine");
+ return std::make_shared<std::string>();
+ }
+
+ std::shared_ptr<std::string> serializedPacketBuffer = std::make_shared<std::string>();
- RpcRequestHeaderProto h;
- h.set_rpckind(RPC_PROTOCOL_BUFFER);
- h.set_rpcop(RpcRequestHeaderProto::RPC_FINAL_PACKET);
- h.set_callid(RpcEngine::kCallIdConnectionContext);
- h.set_clientid(engine_->client_name());
+ RpcRequestHeaderProto headerProto;
+ headerProto.set_rpckind(RPC_PROTOCOL_BUFFER);
+ headerProto.set_rpcop(RpcRequestHeaderProto::RPC_FINAL_PACKET);
+ headerProto.set_callid(RpcEngine::kCallIdConnectionContext);
+ headerProto.set_clientid(pinnedEngine->client_name());
- IpcConnectionContextProto handshake;
- handshake.set_protocol(engine_->protocol_name());
+ IpcConnectionContextProto handshakeContextProto;
+ handshakeContextProto.set_protocol(pinnedEngine->protocol_name());
const std::string & user_name = auth_info_.getUser();
if (!user_name.empty()) {
- *handshake.mutable_userinfo()->mutable_effectiveuser() = user_name;
+ *handshakeContextProto.mutable_userinfo()->mutable_effectiveuser() = user_name;
}
- AddHeadersToPacket(res.get(), {&h, &handshake}, nullptr);
+ AddHeadersToPacket(serializedPacketBuffer.get(), {&headerProto, &handshakeContextProto}, nullptr);
- return res;
+ return serializedPacketBuffer;
}
void RpcConnection::AsyncRpc(
@@ -310,11 +341,22 @@ void RpcConnection::AsyncRpc_locked(
handler(status);
};
- int call_id = (method_name != SASL_METHOD_NAME ? engine_->NextCallId() : RpcEngine::kCallIdSasl);
- auto r = std::make_shared<Request>(engine_, method_name, call_id, req,
- std::move(wrapped_handler));
- auto r_vector = std::vector<std::shared_ptr<Request> > (1, r);
- SendRpcRequests(r_vector);
+
+ std::shared_ptr<Request> rpcRequest;
+ { // Scope to minimize how long RpcEngine's lifetime may be extended
+ std::shared_ptr<LockFreeRpcEngine> pinnedEngine = engine_.lock();
+ if(!pinnedEngine) {
+ LOG_ERROR(kRPC, << "RpcConnection@" << this << " attempted to access invalid RpcEngine");
+ handler(Status::Error("Invalid RpcEngine access."));
+ return;
+ }
+
+ int call_id = (method_name != SASL_METHOD_NAME ? pinnedEngine->NextCallId() : RpcEngine::kCallIdSasl);
+ rpcRequest = std::make_shared<Request>(pinnedEngine, method_name, call_id,
+ req, std::move(wrapped_handler));
+ }
+
+ SendRpcRequests({rpcRequest});
}
void RpcConnection::AsyncRpc(const std::vector<std::shared_ptr<Request> > & requests) {
@@ -330,13 +372,20 @@ void RpcConnection::SendRpcRequests(const std::vector<std::shared_ptr<Request> >
// Oops. The connection failed _just_ before the engine got a chance
// to send it. Register it as a failure
Status status = Status::ResourceUnavailable("RpcConnection closed before send.");
- engine_->AsyncRpcCommsError(status, shared_from_this(), requests);
+
+ std::shared_ptr<LockFreeRpcEngine> pinnedEngine = engine_.lock();
+ if(!pinnedEngine) {
+ LOG_ERROR(kRPC, << "RpcConnection@" << this << " attempted to access invalid RpcEngine");
+ return;
+ }
+
+ pinnedEngine->AsyncRpcCommsError(status, shared_from_this(), requests);
} else {
- for (auto r: requests) {
- if (r->method_name() != SASL_METHOD_NAME)
- pending_requests_.push_back(r);
+ for (auto request : requests) {
+ if (request->method_name() != SASL_METHOD_NAME)
+ pending_requests_.push_back(request);
else
- auth_requests_.push_back(r);
+ auth_requests_.push_back(request);
}
if (connected_ == kConnected || connected_ == kHandshaking || connected_ == kAuthenticating) { // Dont flush if we're waiting or handshaking
FlushPendingRequests();
@@ -395,26 +444,32 @@ void RpcConnection::CommsError(const Status &status) {
// Anything that has been queued to the connection (on the fly or pending)
// will get dinged for a retry
std::vector<std::shared_ptr<Request>> requestsToReturn;
- std::transform(requests_on_fly_.begin(), requests_on_fly_.end(),
+ std::transform(sent_requests_.begin(), sent_requests_.end(),
std::back_inserter(requestsToReturn),
- std::bind(&RequestOnFlyMap::value_type::second, _1));
- requests_on_fly_.clear();
+ std::bind(&SentRequestMap::value_type::second, _1));
+ sent_requests_.clear();
requestsToReturn.insert(requestsToReturn.end(),
std::make_move_iterator(pending_requests_.begin()),
std::make_move_iterator(pending_requests_.end()));
pending_requests_.clear();
- engine_->AsyncRpcCommsError(status, shared_from_this(), requestsToReturn);
+ std::shared_ptr<LockFreeRpcEngine> pinnedEngine = engine_.lock();
+ if(!pinnedEngine) {
+ LOG_ERROR(kRPC, << "RpcConnection@" << this << " attempted to access an invalid RpcEngine");
+ return;
+ }
+
+ pinnedEngine->AsyncRpcCommsError(status, shared_from_this(), requestsToReturn);
}
void RpcConnection::ClearAndDisconnect(const ::asio::error_code &ec) {
Disconnect();
std::vector<std::shared_ptr<Request>> requests;
- std::transform(requests_on_fly_.begin(), requests_on_fly_.end(),
+ std::transform(sent_requests_.begin(), sent_requests_.end(),
std::back_inserter(requests),
- std::bind(&RequestOnFlyMap::value_type::second, _1));
- requests_on_fly_.clear();
+ std::bind(&SentRequestMap::value_type::second, _1));
+ sent_requests_.clear();
requests.insert(requests.end(),
std::make_move_iterator(pending_requests_.begin()),
std::make_move_iterator(pending_requests_.end()));
@@ -426,13 +481,13 @@ void RpcConnection::ClearAndDisconnect(const ::asio::error_code &ec) {
std::shared_ptr<Request> RpcConnection::RemoveFromRunningQueue(int call_id) {
assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
- auto it = requests_on_fly_.find(call_id);
- if (it == requests_on_fly_.end()) {
+ auto it = sent_requests_.find(call_id);
+ if (it == sent_requests_.end()) {
return std::shared_ptr<Request>();
}
auto req = it->second;
- requests_on_fly_.erase(it);
+ sent_requests_.erase(it);
return req;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc5e44d9/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h
index 678dab3..8e579a2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h
@@ -40,7 +40,7 @@ class RpcConnectionImpl : public RpcConnection {
public:
MEMCHECKED_CLASS(RpcConnectionImpl)
- RpcConnectionImpl(RpcEngine *engine);
+ RpcConnectionImpl(std::shared_ptr<RpcEngine> engine);
virtual ~RpcConnectionImpl() override;
virtual void Connect(const std::vector<::asio::ip::tcp::endpoint> &server,
@@ -73,7 +73,7 @@ public:
};
template <class Socket>
-RpcConnectionImpl<Socket>::RpcConnectionImpl(RpcEngine *engine)
+RpcConnectionImpl<Socket>::RpcConnectionImpl(std::shared_ptr<RpcEngine> engine)
: RpcConnection(engine),
options_(engine->options()),
socket_(engine->io_service()),
@@ -88,8 +88,8 @@ RpcConnectionImpl<Socket>::~RpcConnectionImpl() {
if (pending_requests_.size() > 0)
LOG_WARN(kRPC, << "RpcConnectionImpl::~RpcConnectionImpl called with items in the pending queue");
- if (requests_on_fly_.size() > 0)
- LOG_WARN(kRPC, << "RpcConnectionImpl::~RpcConnectionImpl called with items in the requests_on_fly queue");
+ if (sent_requests_.size() > 0)
+ LOG_WARN(kRPC, << "RpcConnectionImpl::~RpcConnectionImpl called with items in the sent_requests queue");
}
template <class Socket>
@@ -101,13 +101,23 @@ void RpcConnectionImpl<Socket>::Connect(
this->auth_info_ = auth_info;
- auto connectionSuccessfulReq = std::make_shared<Request>(
- engine_, [handler](::google::protobuf::io::CodedInputStream *is,
- const Status &status) {
- (void)is;
- handler(status);
- });
- pending_requests_.push_back(connectionSuccessfulReq);
+ std::shared_ptr<Request> connectionRequest;
+ { // Scope to minimize how long RpcEngine's lifetime may be extended
+ std::shared_ptr<LockFreeRpcEngine> pinned_engine = engine_.lock();
+ if(!pinned_engine) {
+ LOG_ERROR(kRPC, << "RpcConnectionImpl@" << this << " attempted to access invalid RpcEngine");
+ handler(Status::Error("Invalid RpcEngine access."));
+ return;
+ }
+
+ connectionRequest = std::make_shared<Request>(pinned_engine,
+ [handler](::google::protobuf::io::CodedInputStream *is,const Status &status) {
+ (void)is;
+ handler(status);
+ });
+ }
+
+ pending_requests_.push_back(connectionRequest);
this->ConnectAndFlush(server); // need "this" so compiler can infer type of CAF
}
@@ -263,7 +273,7 @@ void RpcConnectionImpl<Socket>::OnSendCompleted(const ::asio::error_code &ec,
LOG_TRACE(kRPC, << "RpcConnectionImpl::OnSendCompleted called");
- request_over_the_wire_.reset();
+ outgoing_request_.reset();
if (ec) {
LOG_WARN(kRPC, << "Network error during RPC write: " << ec.message());
CommsError(ToStatus(ec));
@@ -283,7 +293,7 @@ void RpcConnectionImpl<Socket>::FlushPendingRequests() {
LOG_TRACE(kRPC, << "RpcConnectionImpl::FlushPendingRequests called");
// Don't send if we don't need to
- if (request_over_the_wire_) {
+ if (outgoing_request_) {
return;
}
@@ -324,9 +334,9 @@ void RpcConnectionImpl<Socket>::FlushPendingRequests() {
std::shared_ptr<std::string> payload = std::make_shared<std::string>();
req->GetPacket(payload.get());
if (!payload->empty()) {
- assert(requests_on_fly_.find(req->call_id()) == requests_on_fly_.end());
- requests_on_fly_[req->call_id()] = req;
- request_over_the_wire_ = req;
+ assert(sent_requests_.find(req->call_id()) == sent_requests_.end());
+ sent_requests_[req->call_id()] = req;
+ outgoing_request_ = req;
req->timer().expires_from_now(
std::chrono::milliseconds(options_.rpc_timeout));
@@ -343,7 +353,15 @@ void RpcConnectionImpl<Socket>::FlushPendingRequests() {
OnSendCompleted(ec, size);
});
} else { // Nothing to send for this request, inform the handler immediately
- io_service().post(
+ ::asio::io_service *service = GetIoService();
+ if(!service) {
+ LOG_ERROR(kRPC, << "RpcConnectionImpl@" << this << " attempted to access null IoService");
+ // No easy way to bail out of this context, but the only way to get here is when
+ // the FileSystem is being destroyed.
+ return;
+ }
+
+ service->post(
// Never hold locks when calling a callback
[req]() { req->OnResponseArrived(nullptr, Status::OK()); }
);
@@ -433,7 +451,7 @@ void RpcConnectionImpl<Socket>::Disconnect() {
LOG_INFO(kRPC, << "RpcConnectionImpl::Disconnect called");
- request_over_the_wire_.reset();
+ outgoing_request_.reset();
if (connected_ == kConnecting || connected_ == kHandshaking || connected_ == kAuthenticating || connected_ == kConnected) {
// Don't print out errors, we were expecting a disconnect here
SafeDisconnect(get_asio_socket_ptr(&socket_));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc5e44d9/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
index 34af423..0ca7c6a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
@@ -171,7 +171,7 @@ std::shared_ptr<RpcConnection> RpcEngine::NewConnection()
{
LOG_DEBUG(kRPC, << "RpcEngine::NewConnection called");
- return std::make_shared<RpcConnectionImpl<::asio::ip::tcp::socket>>(this);
+ return std::make_shared<RpcConnectionImpl<::asio::ip::tcp::socket>>(shared_from_this());
}
std::shared_ptr<RpcConnection> RpcEngine::InitializeConnection()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc5e44d9/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
index dc4054e..9f45fcf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
@@ -75,16 +75,16 @@ public:
std::vector<std::shared_ptr<Request>> pendingRequests) = 0;
- virtual const RetryPolicy * retry_policy() const = 0;
+ virtual const RetryPolicy *retry_policy() = 0;
virtual int NextCallId() = 0;
- virtual const std::string &client_name() const = 0;
- virtual const std::string &client_id() const = 0;
- virtual const std::string &user_name() const = 0;
- virtual const std::string &protocol_name() const = 0;
- virtual int protocol_version() const = 0;
+ virtual const std::string &client_name() = 0;
+ virtual const std::string &client_id() = 0;
+ virtual const std::string &user_name() = 0;
+ virtual const std::string &protocol_name() = 0;
+ virtual int protocol_version() = 0;
virtual ::asio::io_service &io_service() = 0;
- virtual const Options &options() const = 0;
+ virtual const Options &options() = 0;
};
@@ -95,7 +95,7 @@ public:
* Threading model: thread-safe. All callbacks will be called back from
* an asio pool and will not hold any internal locks
*/
-class RpcEngine : public LockFreeRpcEngine {
+class RpcEngine : public LockFreeRpcEngine, public std::enable_shared_from_this<RpcEngine> {
public:
MEMCHECKED_CLASS(RpcEngine)
enum { kRpcVersion = 9 };
@@ -133,20 +133,20 @@ class RpcEngine : public LockFreeRpcEngine {
std::vector<std::shared_ptr<Request>> pendingRequests);
- const RetryPolicy * retry_policy() const override { return retry_policy_.get(); }
+ const RetryPolicy * retry_policy() override { return retry_policy_.get(); }
int NextCallId() override { return ++call_id_; }
void TEST_SetRpcConnection(std::shared_ptr<RpcConnection> conn);
void TEST_SetRetryPolicy(std::unique_ptr<const RetryPolicy> policy);
std::unique_ptr<const RetryPolicy> TEST_GenerateRetryPolicyUsingOptions();
- const std::string &client_name() const override { return client_name_; }
- const std::string &client_id() const override { return client_id_; }
- const std::string &user_name() const override { return auth_info_.getUser(); }
- const std::string &protocol_name() const override { return protocol_name_; }
- int protocol_version() const override { return protocol_version_; }
+ const std::string &client_name() override { return client_name_; }
+ const std::string &client_id() override { return client_id_; }
+ const std::string &user_name() override { return auth_info_.getUser(); }
+ const std::string &protocol_name() override { return protocol_name_; }
+ int protocol_version() override { return protocol_version_; }
::asio::io_service &io_service() override { return *io_service_; }
- const Options &options() const override { return options_; }
+ const Options &options() override { return options_; }
static std::string GetRandomClientName();
void SetFsEventCallback(fs_event_callback callback);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc5e44d9/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
index 776894e..f998c7f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
@@ -71,7 +71,7 @@ protected:
// Stuff in some dummy endpoints so we don't error out
last_endpoints_ = make_endpoint()[0].endpoints;
- return std::make_shared<RpcConnectionImpl<SharedMockRPCConnection>>(this);
+ return std::make_shared<RpcConnectionImpl<SharedMockRPCConnection>>(shared_from_this());
}
};
@@ -106,9 +106,9 @@ using namespace hdfs;
TEST(RpcEngineTest, TestRoundTrip) {
::asio::io_service io_service;
Options options;
- RpcEngine engine(&io_service, options, "foo", "", "protocol", 1);
+ std::shared_ptr<RpcEngine> engine = std::make_shared<RpcEngine>(&io_service, options, "foo", "", "protocol", 1);
auto conn =
- std::make_shared<RpcConnectionImpl<MockRPCConnection> >(&engine);
+ std::make_shared<RpcConnectionImpl<MockRPCConnection> >(engine);
conn->TEST_set_connected(true);
conn->StartReading();
@@ -122,14 +122,14 @@ TEST(RpcEngineTest, TestRoundTrip) {
.WillOnce(Return(RpcResponse(h, server_resp.SerializeAsString())));
std::shared_ptr<RpcConnection> conn_ptr(conn);
- engine.TEST_SetRpcConnection(conn_ptr);
+ engine->TEST_SetRpcConnection(conn_ptr);
bool complete = false;
EchoRequestProto req;
req.set_message("foo");
std::shared_ptr<EchoResponseProto> resp(new EchoResponseProto());
- engine.AsyncRpc("test", &req, resp, [resp, &complete,&io_service](const Status &stat) {
+ engine->AsyncRpc("test", &req, resp, [resp, &complete,&io_service](const Status &stat) {
ASSERT_TRUE(stat.ok());
ASSERT_EQ("foo", resp->message());
complete = true;
@@ -142,9 +142,9 @@ TEST(RpcEngineTest, TestRoundTrip) {
TEST(RpcEngineTest, TestConnectionResetAndFail) {
::asio::io_service io_service;
Options options;
- RpcEngine engine(&io_service, options, "foo", "", "protocol", 1);
+ std::shared_ptr<RpcEngine> engine = std::make_shared<RpcEngine>(&io_service, options, "foo", "", "protocol", 1);
auto conn =
- std::make_shared<RpcConnectionImpl<MockRPCConnection> >(&engine);
+ std::make_shared<RpcConnectionImpl<MockRPCConnection> >(engine);
conn->TEST_set_connected(true);
conn->StartReading();
@@ -158,13 +158,13 @@ TEST(RpcEngineTest, TestConnectionResetAndFail) {
h, "", make_error_code(::asio::error::connection_reset))));
std::shared_ptr<RpcConnection> conn_ptr(conn);
- engine.TEST_SetRpcConnection(conn_ptr);
+ engine->TEST_SetRpcConnection(conn_ptr);
EchoRequestProto req;
req.set_message("foo");
std::shared_ptr<EchoResponseProto> resp(new EchoResponseProto());
- engine.AsyncRpc("test", &req, resp, [&complete, &io_service](const Status &stat) {
+ engine->AsyncRpc("test", &req, resp, [&complete, &io_service](const Status &stat) {
complete = true;
io_service.stop();
ASSERT_FALSE(stat.ok());
@@ -179,11 +179,12 @@ TEST(RpcEngineTest, TestConnectionResetAndRecover) {
Options options;
options.max_rpc_retries = 1;
options.rpc_retry_delay_ms = 0;
- SharedConnectionEngine engine(&io_service, options, "foo", "", "protocol", 1);
+ std::shared_ptr<SharedConnectionEngine> engine
+ = std::make_shared<SharedConnectionEngine>(&io_service, options, "foo", "", "protocol", 1);
// Normally determined during RpcEngine::Connect, but in this case options
// provides enough info to determine policy here.
- engine.TEST_SetRetryPolicy(engine.TEST_GenerateRetryPolicyUsingOptions());
+ engine->TEST_SetRetryPolicy(engine->TEST_GenerateRetryPolicyUsingOptions());
EchoResponseProto server_resp;
@@ -205,7 +206,7 @@ TEST(RpcEngineTest, TestConnectionResetAndRecover) {
req.set_message("foo");
std::shared_ptr<EchoResponseProto> resp(new EchoResponseProto());
- engine.AsyncRpc("test", &req, resp, [&complete, &io_service](const Status &stat) {
+ engine->AsyncRpc("test", &req, resp, [&complete, &io_service](const Status &stat) {
complete = true;
io_service.stop();
ASSERT_TRUE(stat.ok());
@@ -219,11 +220,12 @@ TEST(RpcEngineTest, TestConnectionResetAndRecoverWithDelay) {
Options options;
options.max_rpc_retries = 1;
options.rpc_retry_delay_ms = 1;
- SharedConnectionEngine engine(&io_service, options, "foo", "", "protocol", 1);
+ std::shared_ptr<SharedConnectionEngine> engine =
+ std::make_shared<SharedConnectionEngine>(&io_service, options, "foo", "", "protocol", 1);
// Normally determined during RpcEngine::Connect, but in this case options
// provides enough info to determine policy here.
- engine.TEST_SetRetryPolicy(engine.TEST_GenerateRetryPolicyUsingOptions());
+ engine->TEST_SetRetryPolicy(engine->TEST_GenerateRetryPolicyUsingOptions());
EchoResponseProto server_resp;
server_resp.set_message("foo");
@@ -244,7 +246,7 @@ TEST(RpcEngineTest, TestConnectionResetAndRecoverWithDelay) {
req.set_message("foo");
std::shared_ptr<EchoResponseProto> resp(new EchoResponseProto());
- engine.AsyncRpc("test", &req, resp, [&complete, &io_service](const Status &stat) {
+ engine->AsyncRpc("test", &req, resp, [&complete, &io_service](const Status &stat) {
complete = true;
io_service.stop();
ASSERT_TRUE(stat.ok());
@@ -272,11 +274,12 @@ TEST(RpcEngineTest, TestConnectionFailure)
Options options;
options.max_rpc_retries = 0;
options.rpc_retry_delay_ms = 0;
- SharedConnectionEngine engine(&io_service, options, "foo", "", "protocol", 1);
+ std::shared_ptr<SharedConnectionEngine> engine
+ = std::make_shared<SharedConnectionEngine>(&io_service, options, "foo", "", "protocol", 1);
EXPECT_CALL(*producer, Produce())
.WillOnce(Return(std::make_pair(make_error_code(::asio::error::connection_reset), "")));
- engine.Connect("", make_endpoint(), [&complete, &io_service](const Status &stat) {
+ engine->Connect("", make_endpoint(), [&complete, &io_service](const Status &stat) {
complete = true;
io_service.stop();
ASSERT_FALSE(stat.ok());
@@ -298,13 +301,14 @@ TEST(RpcEngineTest, TestConnectionFailureRetryAndFailure)
Options options;
options.max_rpc_retries = 2;
options.rpc_retry_delay_ms = 0;
- SharedConnectionEngine engine(&io_service, options, "foo", "", "protocol", 1);
+ std::shared_ptr<SharedConnectionEngine> engine =
+ std::make_shared<SharedConnectionEngine>(&io_service, options, "foo", "", "protocol", 1);
EXPECT_CALL(*producer, Produce())
.WillOnce(Return(std::make_pair(make_error_code(::asio::error::connection_reset), "")))
.WillOnce(Return(std::make_pair(make_error_code(::asio::error::connection_reset), "")))
.WillOnce(Return(std::make_pair(make_error_code(::asio::error::connection_reset), "")));
- engine.Connect("", make_endpoint(), [&complete, &io_service](const Status &stat) {
+ engine->Connect("", make_endpoint(), [&complete, &io_service](const Status &stat) {
complete = true;
io_service.stop();
ASSERT_FALSE(stat.ok());
@@ -326,13 +330,14 @@ TEST(RpcEngineTest, TestConnectionFailureAndRecover)
Options options;
options.max_rpc_retries = 1;
options.rpc_retry_delay_ms = 0;
- SharedConnectionEngine engine(&io_service, options, "foo", "", "protocol", 1);
+ std::shared_ptr<SharedConnectionEngine> engine =
+ std::make_shared<SharedConnectionEngine>(&io_service, options, "foo", "", "protocol", 1);
EXPECT_CALL(*producer, Produce())
.WillOnce(Return(std::make_pair(make_error_code(::asio::error::connection_reset), "")))
.WillOnce(Return(std::make_pair(::asio::error_code(), "")))
.WillOnce(Return(std::make_pair(::asio::error::would_block, "")));
- engine.Connect("", make_endpoint(), [&complete, &io_service](const Status &stat) {
+ engine->Connect("", make_endpoint(), [&complete, &io_service](const Status &stat) {
complete = true;
io_service.stop();
ASSERT_TRUE(stat.ok());
@@ -347,16 +352,17 @@ TEST(RpcEngineTest, TestEventCallbacks)
Options options;
options.max_rpc_retries = 99;
options.rpc_retry_delay_ms = 0;
- SharedConnectionEngine engine(&io_service, options, "foo", "", "protocol", 1);
+ std::shared_ptr<SharedConnectionEngine> engine =
+ std::make_shared<SharedConnectionEngine>(&io_service, options, "foo", "", "protocol", 1);
// Normally determined during RpcEngine::Connect, but in this case options
// provides enough info to determine policy here.
- engine.TEST_SetRetryPolicy(engine.TEST_GenerateRetryPolicyUsingOptions());
+ engine->TEST_SetRetryPolicy(engine->TEST_GenerateRetryPolicyUsingOptions());
// Set up event callbacks
int calls = 0;
std::vector<std::string> callbacks;
- engine.SetFsEventCallback([&calls, &callbacks] (const char * event,
+ engine->SetFsEventCallback([&calls, &callbacks] (const char * event,
const char * cluster,
int64_t value) {
(void)cluster; (void)value;
@@ -393,7 +399,7 @@ TEST(RpcEngineTest, TestEventCallbacks)
std::shared_ptr<EchoResponseProto> resp(new EchoResponseProto());
bool complete = false;
- engine.AsyncRpc("test", &req, resp, [&complete, &io_service](const Status &stat) {
+ engine->AsyncRpc("test", &req, resp, [&complete, &io_service](const Status &stat) {
complete = true;
io_service.stop();
ASSERT_TRUE(stat.ok());
@@ -431,13 +437,14 @@ TEST(RpcEngineTest, TestConnectionFailureAndAsyncRecover)
Options options;
options.max_rpc_retries = 1;
options.rpc_retry_delay_ms = 1;
- SharedConnectionEngine engine(&io_service, options, "foo", "", "protocol", 1);
+ std::shared_ptr<SharedConnectionEngine> engine =
+ std::make_shared<SharedConnectionEngine>(&io_service, options, "foo", "", "protocol", 1);
EXPECT_CALL(*producer, Produce())
.WillOnce(Return(std::make_pair(make_error_code(::asio::error::connection_reset), "")))
.WillOnce(Return(std::make_pair(::asio::error_code(), "")))
.WillOnce(Return(std::make_pair(::asio::error::would_block, "")));
- engine.Connect("", make_endpoint(), [&complete, &io_service](const Status &stat) {
+ engine->Connect("", make_endpoint(), [&complete, &io_service](const Status &stat) {
complete = true;
io_service.stop();
ASSERT_TRUE(stat.ok());
@@ -455,9 +462,9 @@ TEST(RpcEngineTest, TestTimeout) {
::asio::io_service io_service;
Options options;
options.rpc_timeout = 1;
- RpcEngine engine(&io_service, options, "foo", "", "protocol", 1);
+ std::shared_ptr<RpcEngine> engine = std::make_shared<RpcEngine>(&io_service, options, "foo", "", "protocol", 1);
auto conn =
- std::make_shared<RpcConnectionImpl<MockRPCConnection> >(&engine);
+ std::make_shared<RpcConnectionImpl<MockRPCConnection> >(engine);
conn->TEST_set_connected(true);
conn->StartReading();
@@ -465,14 +472,14 @@ TEST(RpcEngineTest, TestTimeout) {
.WillOnce(Return(std::make_pair(::asio::error::would_block, "")));
std::shared_ptr<RpcConnection> conn_ptr(conn);
- engine.TEST_SetRpcConnection(conn_ptr);
+ engine->TEST_SetRpcConnection(conn_ptr);
bool complete = false;
EchoRequestProto req;
req.set_message("foo");
std::shared_ptr<EchoResponseProto> resp(new EchoResponseProto());
- engine.AsyncRpc("test", &req, resp, [resp, &complete,&io_service](const Status &stat) {
+ engine->AsyncRpc("test", &req, resp, [resp, &complete,&io_service](const Status &stat) {
complete = true;
io_service.stop();
ASSERT_FALSE(stat.ok());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/52] [abbrv] hadoop git commit: HDFS-10754: libhdfs++: Create
tools directory and implement hdfs_cat, hdfs_chgrp, hdfs_chown,
hdfs_chmod and hdfs_find. Contributed by Anatoli Shein.
Posted by jh...@apache.org.
HDFS-10754: libhdfs++: Create tools directory and implement hdfs_cat, hdfs_chgrp, hdfs_chown, hdfs_chmod and hdfs_find. Contributed by Anatoli Shein.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4f6cb5d1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4f6cb5d1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4f6cb5d1
Branch: refs/heads/HDFS-8707
Commit: 4f6cb5d1a150b366516ddfc3409925c26a1a2a6a
Parents: 05ddb31
Author: James <jh...@apache.org>
Authored: Fri Aug 26 12:52:05 2016 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../src/main/native/libhdfspp/CMakeLists.txt | 1 +
.../libhdfspp/examples/cpp/CMakeLists.txt | 3 +-
.../native/libhdfspp/examples/cpp/cat/cat.cpp | 126 +++++-----
.../libhdfspp/examples/cpp/find/CMakeLists.txt | 35 +++
.../native/libhdfspp/examples/cpp/find/find.cpp | 162 ++++++++++++
.../libhdfspp/examples/cpp/gendirs/gendirs.cpp | 116 ++++-----
.../native/libhdfspp/include/hdfspp/hdfs_ext.h | 14 ++
.../native/libhdfspp/include/hdfspp/hdfspp.h | 45 +++-
.../native/libhdfspp/include/hdfspp/statinfo.h | 1 +
.../native/libhdfspp/lib/bindings/c/hdfs.cc | 64 +++--
.../lib/common/configuration_loader.cc | 8 +-
.../libhdfspp/lib/common/configuration_loader.h | 8 +-
.../lib/common/configuration_loader_impl.h | 2 +-
.../main/native/libhdfspp/lib/fs/filesystem.cc | 244 +++++++++++++++++--
.../main/native/libhdfspp/lib/fs/filesystem.h | 68 +++++-
.../libhdfspp/lib/fs/namenode_operations.cc | 82 ++-----
.../libhdfspp/lib/fs/namenode_operations.h | 8 +-
.../libhdfspp/tests/configuration_test.cc | 84 +++++--
.../native/libhdfspp/tests/configuration_test.h | 4 +-
.../libhdfspp/tests/hdfs_configuration_test.cc | 9 +-
.../src/main/native/libhdfspp/tests/hdfs_shim.c | 4 +
.../libhdfspp/tests/libhdfs_wrapper_undefs.h | 1 +
.../libhdfspp/tests/libhdfspp_wrapper_defines.h | 1 +
.../main/native/libhdfspp/tools/CMakeLists.txt | 42 ++++
.../main/native/libhdfspp/tools/hdfs_cat.cpp | 120 +++++++++
.../main/native/libhdfspp/tools/hdfs_chgrp.cpp | 196 +++++++++++++++
.../main/native/libhdfspp/tools/hdfs_chmod.cpp | 194 +++++++++++++++
.../main/native/libhdfspp/tools/hdfs_chown.cpp | 206 ++++++++++++++++
.../main/native/libhdfspp/tools/hdfs_find.cpp | 156 ++++++++++++
.../native/libhdfspp/tools/tools_common.cpp | 70 ++++++
.../main/native/libhdfspp/tools/tools_common.h | 39 +++
31 files changed, 1829 insertions(+), 284 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
index a663e60..da56403 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
@@ -163,6 +163,7 @@ add_subdirectory(third_party/uriparser2)
add_subdirectory(lib)
add_subdirectory(tests)
add_subdirectory(examples)
+add_subdirectory(tools)
# create an empty file; hadoop_add_dual_library wraps add_library which
# requires at least one file as an argument
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/CMakeLists.txt
index 183299a..9e16b0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/CMakeLists.txt
@@ -17,4 +17,5 @@
#
add_subdirectory(cat)
-add_subdirectory(gendirs)
\ No newline at end of file
+add_subdirectory(gendirs)
+add_subdirectory(find)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/cat/cat.cpp
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/cat/cat.cpp b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/cat/cat.cpp
index bfab507..17626ea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/cat/cat.cpp
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/cat/cat.cpp
@@ -17,105 +17,91 @@
under the License.
*/
-/*
- A a stripped down version of unix's "cat".
- Doesn't deal with any flags for now, will just attempt to read the whole file.
-*/
+/**
+ * Unix-like cat tool example.
+ *
+ * Reads the specified file from HDFS and outputs to stdout.
+ *
+ * Usage: cat /<path-to-file>
+ *
+ * Example: cat /dir/file
+ *
+ * @param path-to-file Absolute path to the file to read.
+ *
+ **/
#include "hdfspp/hdfspp.h"
#include "common/hdfs_configuration.h"
#include "common/configuration_loader.h"
-#include "common/uri.h"
-
-#include <google/protobuf/io/coded_stream.h>
-using namespace std;
-using namespace hdfs;
+#include <google/protobuf/stubs/common.h>
-#define SCHEME "hdfs"
+#define BUF_SIZE 4096
int main(int argc, char *argv[]) {
if (argc != 2) {
- cerr << "usage: cat [hdfs://[<hostname>:<port>]]/<path-to-file>" << endl;
- return 1;
- }
-
- optional<URI> uri;
- const string uri_path = argv[1];
-
- //Separate check for scheme is required, otherwise common/uri.h library causes memory issues under valgrind
- size_t scheme_end = uri_path.find("://");
- if (scheme_end != string::npos) {
- if(uri_path.substr(0, string(SCHEME).size()).compare(SCHEME) != 0) {
- cerr << "Scheme " << uri_path.substr(0, scheme_end) << ":// is not supported" << endl;
- return 1;
- } else {
- uri = URI::parse_from_string(uri_path);
- }
- }
- if (!uri) {
- cerr << "Malformed URI: " << uri_path << endl;
- return 1;
+ std::cerr << "usage: cat /<path-to-file>" << std::endl;
+ exit(EXIT_FAILURE);
}
-
- ConfigurationLoader loader;
- optional<HdfsConfiguration> config = loader.LoadDefaultResources<HdfsConfiguration>();
- const char * envHadoopConfDir = getenv("HADOOP_CONF_DIR");
- if (envHadoopConfDir && (*envHadoopConfDir != 0) ) {
- config = loader.OverlayResourceFile(*config, string(envHadoopConfDir) + "/core-site.xml");
- }
-
- Options options;
+ std::string path = argv[1];
+
+ hdfs::Options options;
+ //Setting the config path to the default: "$HADOOP_CONF_DIR" or "/etc/hadoop/conf"
+ hdfs::ConfigurationLoader loader;
+ //Loading default config files core-site.xml and hdfs-site.xml from the config path
+ hdfs::optional<hdfs::HdfsConfiguration> config = loader.LoadDefaultResources<hdfs::HdfsConfiguration>();
+ //TODO: HDFS-9539 - after this is resolved, valid config will always be returned.
if(config){
+ //Loading options from the config
options = config->GetOptions();
}
-
- IoService * io_service = IoService::New();
-
- FileSystem *fs_raw = FileSystem::New(io_service, "", options);
- if (!fs_raw) {
- cerr << "Could not create FileSystem object" << endl;
- return 1;
+ hdfs::IoService * io_service = hdfs::IoService::New();
+ //Wrapping fs into a shared pointer to guarantee deletion
+ std::shared_ptr<hdfs::FileSystem> fs(hdfs::FileSystem::New(io_service, "", options));
+ if (!fs) {
+ std::cerr << "Could not connect the file system." << std::endl;
+ exit(EXIT_FAILURE);
}
- //wrapping fs_raw into a unique pointer to guarantee deletion
- unique_ptr<FileSystem> fs(fs_raw);
-
- Status stat = fs->Connect(uri->get_host(), to_string(*(uri->get_port())));
- if (!stat.ok()) {
- cerr << "Could not connect to " << uri->get_host() << ":" << *(uri->get_port()) << endl;
- return 1;
+ hdfs::Status status = fs->ConnectToDefaultFs();
+ if (!status.ok()) {
+ if(!options.defaultFS.get_host().empty()){
+ std::cerr << "Error connecting to " << options.defaultFS << ". " << status.ToString() << std::endl;
+ } else {
+ std::cerr << "Error connecting to the cluster: defaultFS is empty. " << status.ToString() << std::endl;
+ }
+ exit(EXIT_FAILURE);
}
- FileHandle *file_raw = nullptr;
- stat = fs->Open(uri->get_path(), &file_raw);
- if (!stat.ok()) {
- cerr << "Could not open file " << uri->get_path() << endl;
- return 1;
+ hdfs::FileHandle *file_raw = nullptr;
+ status = fs->Open(path, &file_raw);
+ if (!status.ok()) {
+ std::cerr << "Could not open file " << path << ". " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
}
//wrapping file_raw into a unique pointer to guarantee deletion
- unique_ptr<FileHandle> file(file_raw);
+ std::unique_ptr<hdfs::FileHandle> file(file_raw);
- char input_buffer[4096];
- ssize_t read_bytes_count = 0;
- size_t last_read_bytes = 0;
+ char input_buffer[BUF_SIZE];
+ ssize_t total_bytes_read = 0;
+ size_t last_bytes_read = 0;
do{
//Reading file chunks
- Status stat = file->PositionRead(input_buffer, sizeof(input_buffer), read_bytes_count, &last_read_bytes);
- if(stat.ok()) {
+ status = file->Read(input_buffer, sizeof(input_buffer), &last_bytes_read);
+ if(status.ok()) {
//Writing file chunks to stdout
- fwrite(input_buffer, last_read_bytes, 1, stdout);
- read_bytes_count += last_read_bytes;
+ fwrite(input_buffer, last_bytes_read, 1, stdout);
+ total_bytes_read += last_bytes_read;
} else {
- if(stat.is_invalid_offset()){
+ if(status.is_invalid_offset()){
//Reached the end of the file
break;
} else {
- cerr << "Error reading the file: " << stat.ToString() << endl;
- return 1;
+ std::cerr << "Error reading the file: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
}
}
- } while (last_read_bytes > 0);
+ } while (last_bytes_read > 0);
// Clean up static data and prevent valgrind memory leaks
google::protobuf::ShutdownProtobufLibrary();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/find/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/find/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/find/CMakeLists.txt
new file mode 100644
index 0000000..7ae27c5
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/find/CMakeLists.txt
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Default LIBHDFSPP_DIR to the default install location. You can override
+# it by add -DLIBHDFSPP_DIR=... to your cmake invocation
+set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX})
+
+include_directories( ${LIBHDFSPP_DIR}/include )
+link_directories( ${LIBHDFSPP_DIR}/lib )
+
+add_executable(find_cpp find.cpp)
+target_link_libraries(find_cpp hdfspp)
+
+# Several examples in different languages need to produce executables with
+# same names. To allow executables with same names we keep their CMake
+# names different, but specify their executable names as follows:
+set_target_properties( find_cpp
+ PROPERTIES
+ OUTPUT_NAME "find"
+)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/find/find.cpp
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/find/find.cpp b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/find/find.cpp
new file mode 100644
index 0000000..21a731b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/find/find.cpp
@@ -0,0 +1,162 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+/**
+ * A parallel find tool example.
+ *
+ * Finds all files matching the specified name recursively starting from the
+ * specified directory and prints their filepaths. Works either synchronously
+ * or asynchronously.
+ *
+ * Usage: find /<path-to-file> <file-name> <use_async>
+ *
+ * Example: find /dir?/tree* some?file*name 1
+ *
+ * @param path-to-file Absolute path at which to begin search, can have wild
+ * cards and must be non-blank
+ * @param file-name Name to find, can have wild cards and must be non-blank
+ * @param use_async If set to 1 it prints out results asynchronously as
+ * they arrive. If set to 0 results are printed in one
+ * big chunk when it becomes available.
+ *
+ **/
+
+#include "hdfspp/hdfspp.h"
+#include "common/hdfs_configuration.h"
+#include "common/configuration_loader.h"
+
+#include <google/protobuf/stubs/common.h>
+#include <future>
+
+void SyncFind(std::shared_ptr<hdfs::FileSystem> fs, const std::string &path, const std::string &name){
+ std::vector<hdfs::StatInfo> results;
+ //Synchronous call to Find
+ hdfs::Status stat = fs->Find(path, name, hdfs::FileSystem::GetDefaultFindMaxDepth(), &results);
+
+ if (!stat.ok()) {
+ std::cerr << "Error: " << stat.ToString() << std::endl;
+ }
+
+ if(results.empty()){
+ std::cout << "Nothing Found" << std::endl;
+ } else {
+ //Printing out the results
+ for (hdfs::StatInfo const& si : results) {
+ std::cout << si.full_path << std::endl;
+ }
+ }
+}
+
+void AsyncFind(std::shared_ptr<hdfs::FileSystem> fs, const std::string &path, const std::string &name){
+ std::promise<void> promise;
+ std::future<void> future(promise.get_future());
+ bool something_found = false;
+ hdfs::Status status = hdfs::Status::OK();
+
+ /**
+ * Keep requesting more until we get the entire listing. Set the promise
+ * when we have the entire listing to stop.
+ *
+ * Find guarantees that the handler will only be called once at a time,
+ * so we do not need any locking here
+ */
+ auto handler = [&promise, &status, &something_found]
+ (const hdfs::Status &s, const std::vector<hdfs::StatInfo> & si, bool has_more_results) -> bool {
+ //Print result chunks as they arrive
+ if(!si.empty()) {
+ something_found = true;
+ for (hdfs::StatInfo const& s : si) {
+ std::cout << s.full_path << std::endl;
+ }
+ }
+ if(!s.ok() && status.ok()){
+ //We make sure we set 'status' only on the first error.
+ status = s;
+ }
+ if (!has_more_results) {
+ promise.set_value(); //set promise
+ return false; //request stop sending results
+ }
+ return true; //request more results
+ };
+
+ //Asynchronous call to Find
+ fs->Find(path, name, hdfs::FileSystem::GetDefaultFindMaxDepth(), handler);
+
+ //block until promise is set
+ future.get();
+ if(!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ }
+ if(!something_found){
+ std::cout << "Nothing Found" << std::endl;
+ }
+}
+
+int main(int argc, char *argv[]) {
+ if (argc != 4) {
+ std::cerr << "usage: find /<path-to-file> <file-name> <use_async>" << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ std::string path = argv[1];
+ std::string name = argv[2];
+ bool use_async = (std::stoi(argv[3]) != 0);
+
+ hdfs::Options options;
+ //Setting the config path to the default: "$HADOOP_CONF_DIR" or "/etc/hadoop/conf"
+ hdfs::ConfigurationLoader loader;
+ //Loading default config files core-site.xml and hdfs-site.xml from the config path
+ hdfs::optional<hdfs::HdfsConfiguration> config = loader.LoadDefaultResources<hdfs::HdfsConfiguration>();
+ //TODO: HDFS-9539 - after this is resolved, valid config will always be returned.
+ if(config){
+ //Loading options from the config
+ options = config->GetOptions();
+ }
+ //TODO: HDFS-9539 - until then we increase the time-out to allow all recursive async calls to finish
+ options.rpc_timeout = std::numeric_limits<int>::max();
+ hdfs::IoService * io_service = hdfs::IoService::New();
+ //Wrapping fs into a unique pointer to guarantee deletion
+ std::shared_ptr<hdfs::FileSystem> fs(hdfs::FileSystem::New(io_service, "", options));
+ if (!fs) {
+ std::cerr << "Could not connect the file system." << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ hdfs::Status status = fs->ConnectToDefaultFs();
+ if (!status.ok()) {
+ if(!options.defaultFS.get_host().empty()){
+ std::cerr << "Error connecting to " << options.defaultFS << ". " << status.ToString() << std::endl;
+ } else {
+ std::cerr << "Error connecting to the cluster: defaultFS is empty. " << status.ToString() << std::endl;
+ }
+ exit(EXIT_FAILURE);
+ }
+
+ if (use_async){
+ //Example of Async find
+ AsyncFind(fs, path, name);
+ } else {
+ //Example of Sync find
+ SyncFind(fs, path, name);
+ }
+
+ // Clean up static data and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/gendirs/gendirs.cpp
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/gendirs/gendirs.cpp b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/gendirs/gendirs.cpp
index 9f5ae5a..c90abbd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/gendirs/gendirs.cpp
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cpp/gendirs/gendirs.cpp
@@ -23,9 +23,9 @@
* Generates a directory tree with specified depth and fanout starting from
* a given path. Generation is asynchronous.
*
- * Usage: gendirs [hdfs://[<hostname>:<port>]]/<path-to-dir> <depth> <fanout>
+ * Usage: gendirs /<path-to-dir> <depth> <fanout>
*
- * Example: gendirs hdfs://localhost.localdomain:9433/dir0 3 10
+ * Example: gendirs /dir0 3 10
*
* @param path-to-dir Absolute path to the directory tree root where the
* directory tree will be generated
@@ -37,99 +37,76 @@
**/
#include "hdfspp/hdfspp.h"
-#include "fs/namenode_operations.h"
#include "common/hdfs_configuration.h"
#include "common/configuration_loader.h"
-#include "common/uri.h"
-#include <google/protobuf/io/coded_stream.h>
+#include <google/protobuf/stubs/common.h>
#include <future>
-using namespace std;
-using namespace hdfs;
+#define DEFAULT_PERMISSIONS 0755
-#define SCHEME "hdfs"
-
-void GenerateDirectories (shared_ptr<FileSystem> fs, int depth, int level, int fanout, string path, vector<future<Status>> & futures) {
+void GenerateDirectories (std::shared_ptr<hdfs::FileSystem> fs, int depth, int level, int fanout, std::string path, std::vector<std::future<hdfs::Status>> & futures) {
//Level contains our current depth in the directory tree
if(level < depth) {
for(int i = 0; i < fanout; i++){
//Recursive calls to cover all possible paths from the root to the leave nodes
- GenerateDirectories(fs, depth, level+1, fanout, path + "dir" + to_string(i) + "/", futures);
+ GenerateDirectories(fs, depth, level+1, fanout, path + "dir" + std::to_string(i) + "/", futures);
}
} else {
//We have reached the leaf nodes and now start making calls to create directories
//We make a promise which will be set when the call finishes and executes our handler
- auto callstate = make_shared<promise<Status>>();
+ auto callstate = std::make_shared<std::promise<hdfs::Status>>();
//Extract a future from this promise
- future<Status> future(callstate->get_future());
+ std::future<hdfs::Status> future(callstate->get_future());
//Save this future to the vector of futures which will be used to wait on all promises
//after the whole recursion is done
- futures.push_back(move(future));
+ futures.push_back(std::move(future));
//Create a handler that will be executed when Mkdirs is done
- auto handler = [callstate](const Status &s) {
+ auto handler = [callstate](const hdfs::Status &s) {
callstate->set_value(s);
};
//Asynchronous call to create this directory along with all missing parent directories
- fs->Mkdirs(path, NameNodeOperations::GetDefaultPermissionMask(), true, handler);
+ fs->Mkdirs(path, DEFAULT_PERMISSIONS, true, handler);
}
}
int main(int argc, char *argv[]) {
if (argc != 4) {
- cerr << "usage: gendirs [hdfs://[<hostname>:<port>]]/<path-to-dir> <depth> <fanout>" << endl;
- return 1;
+ std::cerr << "usage: gendirs /<path-to-dir> <depth> <fanout>" << std::endl;
+ exit(EXIT_FAILURE);
}
- optional<URI> uri;
- const string uri_path = argv[1];
- const int depth = stoi(argv[2]);
- const int fanout = stoi(argv[3]);
+ std::string path = argv[1];
+ int depth = std::stoi(argv[2]);
+ int fanout = std::stoi(argv[3]);
- //Separate check for scheme is required, otherwise common/uri.h library causes memory issues under valgrind
- size_t scheme_end = uri_path.find("://");
- if (scheme_end != string::npos) {
- if(uri_path.substr(0, string(SCHEME).size()).compare(SCHEME) != 0) {
- cerr << "Scheme " << uri_path.substr(0, scheme_end) << ":// is not supported" << endl;
- return 1;
- } else {
- uri = URI::parse_from_string(uri_path);
- }
- }
- if (!uri) {
- cerr << "Malformed URI: " << uri_path << endl;
- return 1;
- }
-
- ConfigurationLoader loader;
- optional<HdfsConfiguration> config = loader.LoadDefaultResources<HdfsConfiguration>();
- const char * envHadoopConfDir = getenv("HADOOP_CONF_DIR");
- if (envHadoopConfDir && (*envHadoopConfDir != 0) ) {
- config = loader.OverlayResourceFile(*config, string(envHadoopConfDir) + "/core-site.xml");
- }
-
- Options options;
- options.rpc_timeout = numeric_limits<int>::max();
+ hdfs::Options options;
+ //Setting the config path to the default: "$HADOOP_CONF_DIR" or "/etc/hadoop/conf"
+ hdfs::ConfigurationLoader loader;
+ //Loading default config files core-site.xml and hdfs-site.xml from the config path
+ hdfs::optional<hdfs::HdfsConfiguration> config = loader.LoadDefaultResources<hdfs::HdfsConfiguration>();
+ //TODO: HDFS-9539 - after this is resolved, valid config will always be returned.
if(config){
+ //Loading options from the config
options = config->GetOptions();
}
-
- IoService * io_service = IoService::New();
-
- FileSystem *fs_raw = FileSystem::New(io_service, "", options);
- if (!fs_raw) {
- cerr << "Could not create FileSystem object" << endl;
- return 1;
+ //TODO: HDFS-9539 - until then we increase the time-out to allow all recursive async calls to finish
+ options.rpc_timeout = std::numeric_limits<int>::max();
+ hdfs::IoService * io_service = hdfs::IoService::New();
+ //Wrapping fs into a unique pointer to guarantee deletion
+ std::shared_ptr<hdfs::FileSystem> fs(hdfs::FileSystem::New(io_service, "", options));
+ if (!fs) {
+ std::cerr << "Could not connect the file system." << std::endl;
+ exit(EXIT_FAILURE);
}
- //Wrapping fs_raw into a unique pointer to guarantee deletion
- shared_ptr<FileSystem> fs(fs_raw);
-
- //Get port from the uri, otherwise use the default port
- string port = to_string(uri->get_port().value_or(8020));
- Status stat = fs->Connect(uri->get_host(), port);
- if (!stat.ok()) {
- cerr << "Could not connect to " << uri->get_host() << ":" << port << endl;
- return 1;
+ hdfs::Status status = fs->ConnectToDefaultFs();
+ if (!status.ok()) {
+ if(!options.defaultFS.get_host().empty()){
+ std::cerr << "Error connecting to " << options.defaultFS << ". " << status.ToString() << std::endl;
+ } else {
+ std::cerr << "Error connecting to the cluster: defaultFS is empty. " << status.ToString() << std::endl;
+ }
+ exit(EXIT_FAILURE);
}
/**
@@ -143,22 +120,23 @@ int main(int argc, char *argv[]) {
* processed. After the whole recursion is complete we will need to wait until
* all promises are set before we can exit.
**/
- vector<future<Status>> futures;
+ std::vector<std::future<hdfs::Status>> futures;
- GenerateDirectories(fs, depth, 0, fanout, uri->get_path() + "/", futures);
+ GenerateDirectories(fs, depth, 0, fanout, path + "/", futures);
/**
* We are waiting here until all promises are set, and checking whether
* the returned statuses contained any errors.
**/
- for(future<Status> &fs : futures){
- Status stat = fs.get();
- if (!stat.ok()) {
- cerr << "Error: " << stat.ToString() << endl;
+ for(std::future<hdfs::Status> &fs : futures){
+ hdfs::Status status = fs.get();
+ if (!status.ok()) {
+ std::cerr << "Error: " << status.ToString() << std::endl;
+ exit(EXIT_FAILURE);
}
}
- cout << "All done!" << endl;
+ std::cout << "All done!" << std::endl;
// Clean up static data and prevent valgrind memory leaks
google::protobuf::ShutdownProtobufLibrary();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
index ce9f0f5..b41857c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
@@ -287,6 +287,20 @@ LIBHDFS_EXTERNAL
int hdfsPreAttachFileMonitor(libhdfspp_file_event_callback handler, int64_t cookie);
+/**
+ * Finds file name on the file system. hdfsFreeFileInfo should be called to deallocate memory.
+ *
+ * @param fs The filesystem (required)
+ * @param path Path at which to begin search, can have wild cards (must be non-blank)
+ * @param name Name to find, can have wild cards (must be non-blank)
+ * @param numEntries Set to the number of files/directories in the result.
+ * @return Returns a dynamically-allocated array of hdfsFileInfo
+ * objects; NULL on error or empty result.
+ * errno is set to non-zero on error or zero on success.
+ **/
+hdfsFileInfo * hdfsFind(hdfsFS fs, const char* path, const char* name, uint32_t * numEntries);
+
+
/*****************************************************************************
* HDFS SNAPSHOT FUNCTIONS
****************************************************************************/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
index 46fe8e9..4b88fe5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
@@ -138,6 +138,18 @@ public:
**/
class FileSystem {
public:
+ //Returns the default maximum depth for recursive Find tool
+ static uint32_t GetDefaultFindMaxDepth();
+
+ //Returns the default permission mask
+ static uint16_t GetDefaultPermissionMask();
+
+ //Checks if the given permission mask is valid
+ static Status CheckValidPermissionMask(uint16_t permissions);
+
+ //Checks if replication value is valid
+ static Status CheckValidReplication(uint16_t replication);
+
/**
* Create a new instance of the FileSystem object. The call
* initializes the RPC connections to the NameNode and returns an
@@ -236,7 +248,7 @@ class FileSystem {
*
* The asynchronous method will return batches of files; the consumer must
* return true if they want more files to be delivered. The final bool
- * parameter in the callback will be set to true if this is the final
+ * parameter in the callback will be set to false if this is the final
* batch of files.
*
* The synchronous method will return all files in the directory.
@@ -245,9 +257,8 @@ class FileSystem {
**/
virtual void
GetListing(const std::string &path,
- const std::function<bool(const Status &, std::shared_ptr<std::vector<StatInfo>> &, bool)> &handler) = 0;
- virtual Status GetListing(const std::string &path,
- std::shared_ptr<std::vector<StatInfo>> & stat_infos) = 0;
+ const std::function<bool(const Status &, const std::vector<StatInfo> &, bool)> &handler) = 0;
+ virtual Status GetListing(const std::string &path, std::vector<StatInfo> * stat_infos) = 0;
/**
* Returns the locations of all known blocks for the indicated file (or part of it), or an error
@@ -297,8 +308,8 @@ class FileSystem {
* @param path the path to the file or directory
* @param permissions the bitmask to set it to (should be between 0 and 01777)
*/
- virtual void SetPermission(const std::string & path,
- uint16_t permissions, const std::function<void(const Status &)> &handler) = 0;
+ virtual void SetPermission(const std::string & path, uint16_t permissions,
+ const std::function<void(const Status &)> &handler) = 0;
virtual Status SetPermission(const std::string & path, uint16_t permissions) = 0;
/**
@@ -307,12 +318,34 @@ class FileSystem {
* @param path file path
* @param username If it is empty, the original username remains unchanged.
* @param groupname If it is empty, the original groupname remains unchanged.
+ * @param recursive If true, the change will be propagated recursively.
*/
virtual void SetOwner(const std::string & path, const std::string & username,
const std::string & groupname, const std::function<void(const Status &)> &handler) = 0;
virtual Status SetOwner(const std::string & path,
const std::string & username, const std::string & groupname) = 0;
+ /**
+ * Finds all files matching the specified name recursively starting from the
+ * specified directory. Returns metadata for each of them.
+ *
+ * Example: Find("/dir?/tree*", "some?file*name")
+ *
+ * @param path Absolute path at which to begin search, can have wild cards (must be non-blank)
+ * @param name Name to find, can also have wild cards (must be non-blank)
+ *
+ * The asynchronous method will return batches of files; the consumer must
+ * return true if they want more files to be delivered. The final bool
+ * parameter in the callback will be set to false if this is the final
+ * batch of files.
+ *
+ * The synchronous method will return matching files.
+ **/
+ virtual void
+ Find(const std::string &path, const std::string &name, const uint32_t maxdepth,
+ const std::function<bool(const Status &, const std::vector<StatInfo> & , bool)> &handler) = 0;
+ virtual Status Find(const std::string &path, const std::string &name,
+ const uint32_t maxdepth, std::vector<StatInfo> * stat_infos) = 0;
/*****************************************************************************
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h
index a53ab8b..e077dda 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h
@@ -33,6 +33,7 @@ struct StatInfo {
int file_type;
::std::string path;
+ ::std::string full_path;
unsigned long int length;
unsigned long int permissions; //Octal number as in POSIX permissions; e.g. 0777
::std::string owner;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
index 4003358..a43d94f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
@@ -731,22 +731,21 @@ hdfsFileInfo *hdfsListDirectory(hdfsFS fs, const char* path, int *numEntries) {
if(!abs_path) {
return nullptr;
}
- std::shared_ptr<std::vector<StatInfo>> stat_infos;
- Status stat = fs->get_impl()->GetListing(*abs_path, stat_infos);
+ std::vector<StatInfo> stat_infos;
+ Status stat = fs->get_impl()->GetListing(*abs_path, &stat_infos);
if (!stat.ok()) {
Error(stat);
*numEntries = 0;
return nullptr;
}
- //Existing API expects nullptr if size is 0
- if(!stat_infos || stat_infos->size()==0){
+ if(stat_infos.empty()){
*numEntries = 0;
return nullptr;
}
- *numEntries = stat_infos->size();
- hdfsFileInfo *file_infos = new hdfsFileInfo[stat_infos->size()];
- for(std::vector<StatInfo>::size_type i = 0; i < stat_infos->size(); i++) {
- StatInfoToHdfsFileInfo(&file_infos[i], stat_infos->at(i));
+ *numEntries = stat_infos.size();
+ hdfsFileInfo *file_infos = new hdfsFileInfo[stat_infos.size()];
+ for(std::vector<StatInfo>::size_type i = 0; i < stat_infos.size(); i++) {
+ StatInfoToHdfsFileInfo(&file_infos[i], stat_infos.at(i));
}
return file_infos;
@@ -785,7 +784,7 @@ int hdfsCreateDirectory(hdfsFS fs, const char* path) {
}
Status stat;
//Use default permissions and set true for creating all non-existant parent directories
- stat = fs->get_impl()->Mkdirs(*abs_path, NameNodeOperations::GetDefaultPermissionMask(), true);
+ stat = fs->get_impl()->Mkdirs(*abs_path, FileSystem::GetDefaultPermissionMask(), true);
if (!stat.ok()) {
return Error(stat);
}
@@ -854,7 +853,7 @@ int hdfsChmod(hdfsFS fs, const char* path, short mode){
if(!abs_path) {
return -1;
}
- Status stat = NameNodeOperations::CheckValidPermissionMask(mode);
+ Status stat = FileSystem::CheckValidPermissionMask(mode);
if (!stat.ok()) {
return Error(stat);
}
@@ -896,6 +895,44 @@ int hdfsChown(hdfsFS fs, const char* path, const char *owner, const char *group)
}
}
+hdfsFileInfo * hdfsFind(hdfsFS fs, const char* path, const char* name, uint32_t * numEntries){
+ try {
+ errno = 0;
+ if (!CheckSystem(fs)) {
+ *numEntries = 0;
+ return nullptr;
+ }
+
+ std::vector<StatInfo> stat_infos;
+ Status stat = fs->get_impl()->Find(path, name, hdfs::FileSystem::GetDefaultFindMaxDepth(), &stat_infos);
+ if (!stat.ok()) {
+ Error(stat);
+ *numEntries = 0;
+ return nullptr;
+ }
+ //Existing API expects nullptr if size is 0
+ if(stat_infos.empty()){
+ *numEntries = 0;
+ return nullptr;
+ }
+ *numEntries = stat_infos.size();
+ hdfsFileInfo *file_infos = new hdfsFileInfo[stat_infos.size()];
+ for(std::vector<StatInfo>::size_type i = 0; i < stat_infos.size(); i++) {
+ StatInfoToHdfsFileInfo(&file_infos[i], stat_infos.at(i));
+ }
+
+ return file_infos;
+ } catch (const std::exception & e) {
+ ReportException(e);
+ *numEntries = 0;
+ return nullptr;
+ } catch (...) {
+ ReportCaughtNonException();
+ *numEntries = 0;
+ return nullptr;
+ }
+}
+
int hdfsCreateSnapshot(hdfsFS fs, const char* path, const char* name) {
try {
errno = 0;
@@ -1373,19 +1410,18 @@ HdfsConfiguration LoadDefault(ConfigurationLoader & loader)
}
else
{
- return loader.New<HdfsConfiguration>();
+ return loader.NewConfig<HdfsConfiguration>();
}
}
-hdfsBuilder::hdfsBuilder() : config(loader.New<HdfsConfiguration>())
+hdfsBuilder::hdfsBuilder() : config(loader.NewConfig<HdfsConfiguration>())
{
errno = 0;
- loader.SetDefaultSearchPath();
config = LoadDefault(loader);
}
hdfsBuilder::hdfsBuilder(const char * directory) :
- config(loader.New<HdfsConfiguration>())
+ config(loader.NewConfig<HdfsConfiguration>())
{
errno = 0;
loader.SetSearchPath(directory);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.cc
index 1eb70c3..e1434dad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.cc
@@ -62,6 +62,12 @@ bool str_to_bool(const std::string& raw) {
return false;
}
+ConfigurationLoader::ConfigurationLoader() {
+ //In order to creates a configuration loader with the default search path
+ //("$HADOOP_CONF_DIR" or "/etc/hadoop/conf") we call SetDefaultSearchPath().
+ ConfigurationLoader::SetDefaultSearchPath();
+}
+
void ConfigurationLoader::SetDefaultSearchPath() {
// Try (in order, taking the first valid one):
// $HADOOP_CONF_DIR
@@ -257,4 +263,4 @@ bool ConfigurationLoader::UpdateMapWithValue(ConfigMap& map,
return true;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.h
index 059e48b..51ac23a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.h
@@ -27,9 +27,9 @@ namespace hdfs {
class ConfigurationLoader {
public:
// Creates a new, empty Configuration object
- // T must be Configuration or a subclass
+ // T must be Configuration or a subclass
template<class T>
- T New();
+ T NewConfig();
/****************************************************************************
* LOADING CONFIG FILES
@@ -79,6 +79,10 @@ public:
* SEARCH PATH METHODS
***************************************************************************/
+ //Creates a configuration loader with the default search path ("$HADOOP_CONF_DIR" or "/etc/hadoop/conf").
+ //If you want to explicitly set the entire search path, call ClearSearchPath() first
+ ConfigurationLoader();
+
// Sets the search path to the default search path (namely, "$HADOOP_CONF_DIR" or "/etc/hadoop/conf")
void SetDefaultSearchPath();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader_impl.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader_impl.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader_impl.h
index 9e18878..6258450 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader_impl.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader_impl.h
@@ -23,7 +23,7 @@ namespace hdfs {
template<class T>
-T ConfigurationLoader::New() {
+T ConfigurationLoader::NewConfig() {
return T();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
index de6ebb7..d75939f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
@@ -26,6 +26,7 @@
#include <tuple>
#include <iostream>
#include <pwd.h>
+#include <fnmatch.h>
#define FMT_THIS_ADDR "this=" << (void*)this
@@ -39,6 +40,34 @@ using ::asio::ip::tcp;
static constexpr uint16_t kDefaultPort = 8020;
+uint32_t FileSystem::GetDefaultFindMaxDepth() {
+ return std::numeric_limits<uint32_t>::max();
+}
+
+uint16_t FileSystem::GetDefaultPermissionMask() {
+ return 0755;
+}
+
+Status FileSystem::CheckValidPermissionMask(uint16_t permissions) {
+ if (permissions > 01777) {
+ std::stringstream errormsg;
+ errormsg << "CheckValidPermissionMask: argument 'permissions' is " << std::oct
+ << std::showbase << permissions << " (should be between 0 and 01777)";
+ return Status::InvalidArgument(errormsg.str().c_str());
+ }
+ return Status::OK();
+}
+
+Status FileSystem::CheckValidReplication(uint16_t replication) {
+ if (replication < 1 || replication > 512) {
+ std::stringstream errormsg;
+ errormsg << "CheckValidReplication: argument 'replication' is "
+ << replication << " (should be between 1 and 512)";
+ return Status::InvalidArgument(errormsg.str().c_str());
+ }
+ return Status::OK();
+}
+
/*****************************************************************************
* FILESYSTEM BASE CLASS
****************************************************************************/
@@ -446,7 +475,7 @@ void FileSystemImpl::SetReplication(const std::string & path, int16_t replicatio
handler(Status::InvalidArgument("SetReplication: argument 'path' cannot be empty"));
return;
}
- Status replStatus = NameNodeOperations::CheckValidReplication(replication);
+ Status replStatus = FileSystem::CheckValidReplication(replication);
if (!replStatus.ok()) {
handler(replStatus);
return;
@@ -593,44 +622,43 @@ Status FileSystemImpl::GetFsStats(FsInfo & fs_info) {
* Some compilers don't like recursive lambdas, so we make the lambda call a
* method, which in turn creates a lambda calling itself.
*/
-void FileSystemImpl::GetListingShim(const Status &stat, std::shared_ptr<std::vector<StatInfo>> &stat_infos, bool has_more,
- std::string path,
- const std::function<bool(const Status &, std::shared_ptr<std::vector<StatInfo>>&, bool)> &handler) {
- bool has_next = stat_infos && stat_infos->size() > 0;
+void FileSystemImpl::GetListingShim(const Status &stat, const std::vector<StatInfo> & stat_infos, bool has_more,
+ std::string path, const std::function<bool(const Status &, const std::vector<StatInfo> &, bool)> &handler) {
+ bool has_next = !stat_infos.empty();
bool get_more = handler(stat, stat_infos, has_more && has_next);
if (get_more && has_more && has_next ) {
- auto callback = [this, path, handler](const Status &stat, std::shared_ptr<std::vector<StatInfo>> &stat_infos, bool has_more) {
+ auto callback = [this, path, handler](const Status &stat, const std::vector<StatInfo> & stat_infos, bool has_more) {
GetListingShim(stat, stat_infos, has_more, path, handler);
};
- std::string last = stat_infos->back().path;
+ std::string last = stat_infos.back().path;
nn_.GetListing(path, callback, last);
}
}
void FileSystemImpl::GetListing(
const std::string &path,
- const std::function<bool(const Status &, std::shared_ptr<std::vector<StatInfo>>&, bool)> &handler) {
+ const std::function<bool(const Status &, const std::vector<StatInfo> &, bool)> &handler) {
LOG_INFO(kFileSystem, << "FileSystemImpl::GetListing("
<< FMT_THIS_ADDR << ", path="
<< path << ") called");
// Caputure the state and push it into the shim
- auto callback = [this, path, handler](const Status &stat, std::shared_ptr<std::vector<StatInfo>> &stat_infos, bool has_more) {
+ auto callback = [this, path, handler](const Status &stat, const std::vector<StatInfo> & stat_infos, bool has_more) {
GetListingShim(stat, stat_infos, has_more, path, handler);
};
nn_.GetListing(path, callback);
}
-Status FileSystemImpl::GetListing(const std::string &path, std::shared_ptr<std::vector<StatInfo>> &stat_infos) {
+Status FileSystemImpl::GetListing(const std::string &path, std::vector<StatInfo> * stat_infos) {
LOG_INFO(kFileSystem, << "FileSystemImpl::[sync]GetListing("
<< FMT_THIS_ADDR << ", path="
<< path << ") called");
- // In this case, we're going to allocate the result on the heap and have the
- // async code populate it.
- auto results = std::make_shared<std::vector<StatInfo>>();
+ if (!stat_infos) {
+ return Status::InvalidArgument("FileSystemImpl::GetListing: argument 'stat_infos' cannot be NULL");
+ }
auto callstate = std::make_shared<std::promise<Status>>();
std::future<Status> future(callstate->get_future());
@@ -640,9 +668,9 @@ Status FileSystemImpl::GetListing(const std::string &path, std::shared_ptr<std::
Keep requesting more until we get the entire listing, and don't set the promise
* until we have the entire listing.
*/
- auto h = [callstate, results](const Status &s, std::shared_ptr<std::vector<StatInfo>> si, bool has_more) -> bool {
- if (si) {
- results->insert(results->end(), si->begin(), si->end());
+ auto h = [callstate, stat_infos](const Status &s, const std::vector<StatInfo> & si, bool has_more) -> bool {
+ if (!si.empty()) {
+ stat_infos->insert(stat_infos->end(), si.begin(), si.end());
}
bool done = !s.ok() || !has_more;
@@ -658,11 +686,6 @@ Status FileSystemImpl::GetListing(const std::string &path, std::shared_ptr<std::
/* block until promise is set */
Status stat = future.get();
- if (!stat.ok()) {
- return stat;
- }
-
- stat_infos = results;
return stat;
}
@@ -677,7 +700,7 @@ void FileSystemImpl::Mkdirs(const std::string & path, uint16_t permissions, bool
return;
}
- Status permStatus = NameNodeOperations::CheckValidPermissionMask(permissions);
+ Status permStatus = FileSystem::CheckValidPermissionMask(permissions);
if (!permStatus.ok()) {
handler(permStatus);
return;
@@ -790,7 +813,7 @@ void FileSystemImpl::SetPermission(const std::string & path,
handler(Status::InvalidArgument("SetPermission: argument 'path' cannot be empty"));
return;
}
- Status permStatus = NameNodeOperations::CheckValidPermissionMask(permissions);
+ Status permStatus = FileSystem::CheckValidPermissionMask(permissions);
if (!permStatus.ok()) {
handler(permStatus);
return;
@@ -832,8 +855,8 @@ void FileSystemImpl::SetOwner(const std::string & path, const std::string & user
nn_.SetOwner(path, username, groupname, handler);
}
-Status FileSystemImpl::SetOwner(const std::string & path,
- const std::string & username, const std::string & groupname) {
+Status FileSystemImpl::SetOwner(const std::string & path, const std::string & username,
+ const std::string & groupname) {
LOG_DEBUG(kFileSystem,
<< "FileSystemImpl::[sync]SetOwner(" << FMT_THIS_ADDR << ", path=" << path << ", username=" << username << ", groupname=" << groupname << ") called");
@@ -849,10 +872,179 @@ Status FileSystemImpl::SetOwner(const std::string & path,
/* block until promise is set */
Status stat = future.get();
-
return stat;
}
+/**
+ * Helper function for recursive Find calls.
+ *
+ * Some compilers don't like recursive lambdas, so we make the lambda call a
+ * method, which in turn creates a lambda calling itself.
+ *
+ * ***High-level explanation***
+ *
+ * Since we are allowing to use wild cards in both path and name, we start by expanding the path first.
+ * Boolean search_path is set to true when we search for the path and false when we search for the name.
+ * When we search for the path we break the given path pattern into sub-directories. Starting from the
+ * first sub-directory we list them one-by-one and recursively continue into directories that matched the
+ * path pattern at the current depth. Directories that are large will be requested to continue sending
+ * the results. We keep track of the current depth within the path pattern in the 'depth' variable.
+ * This continues recursively until the depth reaches the end of the path. Next that we start matching
+ * the name pattern. All directories that we find we recurse now, and all names that match the given name
+ * pattern are being stored in outputs and later sent back to the user.
+ */
+void FileSystemImpl::FindShim(const Status &stat, const std::vector<StatInfo> & stat_infos, bool directory_has_more,
+ std::shared_ptr<FindOperationalState> operational_state, std::shared_ptr<FindSharedState> shared_state) {
+ //We buffer the outputs then send them back at the end
+ std::vector<StatInfo> outputs;
+ //Return on error
+ if(!stat.ok()){
+ std::lock_guard<std::mutex> find_lock(shared_state->lock);
+ //We send true becuase we do not want the user code to exit before all our requests finished
+ shared_state->handler(stat, outputs, true);
+ shared_state->aborted = true;
+ }
+ if(!shared_state->aborted){
+ //User did not abort the operation
+ if (directory_has_more) {
+ //Directory is large and has more results
+ //We launch another async call to get more results
+ shared_state->outstanding_requests++;
+ auto callback = [this, operational_state, shared_state](const Status &stat, const std::vector<StatInfo> & stat_infos, bool has_more) {
+ FindShim(stat, stat_infos, has_more, operational_state, shared_state);
+ };
+ std::string last = stat_infos.back().path;
+ nn_.GetListing(operational_state->path, callback, last);
+ }
+ if(operational_state->search_path && operational_state->depth < shared_state->dirs.size() - 1){
+ //We are searching for the path and did not reach the end of the path yet
+ for (StatInfo const& si : stat_infos) {
+ //If we are at the last depth and it matches both path and name, we need to output it.
+ if (operational_state->depth == shared_state->dirs.size() - 2
+ && !fnmatch(shared_state->dirs[operational_state->depth + 1].c_str(), si.path.c_str(), 0)
+ && !fnmatch(shared_state->name.c_str(), si.path.c_str(), 0)) {
+ outputs.push_back(si);
+ }
+ //Skip if not directory
+ if(si.file_type != StatInfo::IS_DIR) {
+ continue;
+ }
+ //Checking for a match with the path at the current depth
+ if(!fnmatch(shared_state->dirs[operational_state->depth + 1].c_str(), si.path.c_str(), 0)){
+ //Launch a new requests for every matched directory
+ shared_state->outstanding_requests++;
+ auto callback = [this, si, operational_state, shared_state](const Status &stat, const std::vector<StatInfo> & stat_infos, bool has_more) {
+ std::shared_ptr<FindOperationalState> new_current_state = std::make_shared<FindOperationalState>(si.full_path, operational_state->depth + 1, true); //true because searching for the path
+ FindShim(stat, stat_infos, has_more, new_current_state, shared_state);
+ };
+ nn_.GetListing(si.full_path, callback);
+ }
+ }
+ }
+ else if(shared_state->maxdepth > operational_state->depth - shared_state->dirs.size() + 1){
+ //We are searching for the name now and maxdepth has not been reached
+ for (StatInfo const& si : stat_infos) {
+ //Launch a new request for every directory
+ if(si.file_type == StatInfo::IS_DIR) {
+ shared_state->outstanding_requests++;
+ auto callback = [this, si, operational_state, shared_state](const Status &stat, const std::vector<StatInfo> & stat_infos, bool has_more) {
+ std::shared_ptr<FindOperationalState> new_current_state = std::make_shared<FindOperationalState>(si.full_path, operational_state->depth + 1, false); //false because searching for the name
+ FindShim(stat, stat_infos, has_more, new_current_state, shared_state);
+ };
+ nn_.GetListing(si.full_path, callback);
+ }
+ //All names that match the specified name are saved to outputs
+ if(!fnmatch(shared_state->name.c_str(), si.path.c_str(), 0)){
+ outputs.push_back(si);
+ }
+ }
+ }
+ }
+ //This section needs a lock to make sure we return the final chunk only once
+ //and no results are sent after aborted is set
+ std::lock_guard<std::mutex> find_lock(shared_state->lock);
+ //Decrement the counter once since we are done with this chunk
+ shared_state->outstanding_requests--;
+ if(shared_state->outstanding_requests == 0){
+ //Send the outputs back to the user and notify that this is the final chunk
+ shared_state->handler(stat, outputs, false);
+ } else {
+ //There will be more results and we are not aborting
+ if (outputs.size() > 0 && !shared_state->aborted){
+ //Send the outputs back to the user and notify that there is more
+ bool user_wants_more = shared_state->handler(stat, outputs, true);
+ if(!user_wants_more) {
+ //Abort if user doesn't want more
+ shared_state->aborted = true;
+ }
+ }
+ }
+}
+
+void FileSystemImpl::Find(
+ const std::string &path, const std::string &name, const uint32_t maxdepth,
+ const std::function<bool(const Status &, const std::vector<StatInfo> &, bool)> &handler) {
+ LOG_INFO(kFileSystem, << "FileSystemImpl::Find("
+ << FMT_THIS_ADDR << ", path="
+ << path << ", name="
+ << name << ") called");
+
+ //Populating the operational state, which includes:
+ //current search path, depth within the path, and the indication that we are currently searching for a path (not name yet).
+ std::shared_ptr<FindOperationalState> operational_state = std::make_shared<FindOperationalState>(path, 0, true);
+ //Populating the shared state, which includes:
+ //vector of sub-directories constructed from path, name to search, handler to use for result returning, outstanding_requests counter, and aborted flag.
+ std::shared_ptr<FindSharedState> shared_state = std::make_shared<FindSharedState>(path, name, maxdepth, handler, 1, false);
+ auto callback = [this, operational_state, shared_state](const Status &stat, const std::vector<StatInfo> & stat_infos, bool directory_has_more) {
+ FindShim(stat, stat_infos, directory_has_more, operational_state, shared_state);
+ };
+ nn_.GetListing("/", callback);
+}
+
+Status FileSystemImpl::Find(const std::string &path, const std::string &name, const uint32_t maxdepth, std::vector<StatInfo> * stat_infos) {
+ LOG_INFO(kFileSystem, << "FileSystemImpl::[sync]Find("
+ << FMT_THIS_ADDR << ", path="
+ << path << ", name="
+ << name << ") called");
+
+ if (!stat_infos) {
+ return Status::InvalidArgument("FileSystemImpl::Find: argument 'stat_infos' cannot be NULL");
+ }
+
+ // In this case, we're going to have the async code populate stat_infos.
+
+ std::promise<void> promise = std::promise<void>();
+ std::future<void> future(promise.get_future());
+ Status status = Status::OK();
+
+ /**
+ * Keep requesting more until we get the entire listing. Set the promise
+ * when we have the entire listing to stop.
+ *
+ * Find guarantees that the handler will only be called once at a time,
+ * so we do not need any locking here
+ */
+ auto h = [&status, &promise, stat_infos](const Status &s, const std::vector<StatInfo> & si, bool has_more_results) -> bool {
+ if (!si.empty()) {
+ stat_infos->insert(stat_infos->end(), si.begin(), si.end());
+ }
+ if (!s.ok() && status.ok()){
+ //We make sure we set 'status' only on the first error.
+ status = s;
+ }
+ if (!has_more_results) {
+ promise.set_value();
+ return false;
+ }
+ return true;
+ };
+
+ Find(path, name, maxdepth, h);
+
+ /* block until promise is set */
+ future.get();
+ return status;
+}
void FileSystemImpl::CreateSnapshot(const std::string &path,
const std::string &name,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
index 75a43f3..0e9cedd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
@@ -94,9 +94,9 @@ public:
void GetListing(
const std::string &path,
- const std::function<bool(const Status &, std::shared_ptr<std::vector<StatInfo>> &, bool)> &handler) override;
+ const std::function<bool(const Status &, const std::vector<StatInfo> &, bool)> &handler) override;
- Status GetListing(const std::string &path, std::shared_ptr<std::vector<StatInfo>> &stat_infos) override;
+ Status GetListing(const std::string &path, std::vector<StatInfo> * stat_infos) override;
virtual void GetBlockLocations(const std::string & path, uint64_t offset, uint64_t length,
const std::function<void(const Status &, std::shared_ptr<FileBlockLocation> locations)> ) override;
@@ -115,8 +115,8 @@ public:
const std::function<void(const Status &)> &handler) override;
virtual Status Rename(const std::string &oldPath, const std::string &newPath) override;
- virtual void SetPermission(const std::string & path,
- uint16_t permissions, const std::function<void(const Status &)> &handler) override;
+ virtual void SetPermission(const std::string & path, uint16_t permissions,
+ const std::function<void(const Status &)> &handler) override;
virtual Status SetPermission(const std::string & path, uint16_t permissions) override;
virtual void SetOwner(const std::string & path, const std::string & username,
@@ -124,6 +124,11 @@ public:
virtual Status SetOwner(const std::string & path,
const std::string & username, const std::string & groupname) override;
+ void Find(
+ const std::string &path, const std::string &name, const uint32_t maxdepth,
+ const std::function<bool(const Status &, const std::vector<StatInfo> &, bool)> &handler) override;
+ Status Find(const std::string &path, const std::string &name, const uint32_t maxdepth, std::vector<StatInfo> * stat_infos) override;
+
/*****************************************************************************
* FILE SYSTEM SNAPSHOT FUNCTIONS
****************************************************************************/
@@ -204,9 +209,58 @@ private:
**/
std::shared_ptr<LibhdfsEvents> event_handlers_;
- void GetListingShim(const Status &stat, std::shared_ptr<std::vector<StatInfo>> &stat_infos, bool has_more,
- std::string path,
- const std::function<bool(const Status &, std::shared_ptr<std::vector<StatInfo>>&, bool)> &handler);
+ void GetListingShim(const Status &stat, const std::vector<StatInfo> & stat_infos, bool has_more,
+ std::string path, const std::function<bool(const Status &, const std::vector<StatInfo> &, bool)> &handler);
+
+ struct FindSharedState {
+ //Name pattern (can have wild-cards) to find
+ const std::string name;
+ //Maximum depth to recurse after the end of path is reached.
+ //Can be set to 0 for pure path globbing and ignoring name pattern entirely.
+ const uint32_t maxdepth;
+ //Vector of all sub-directories from the path argument (each can have wild-cards)
+ std::vector<std::string> dirs;
+ //Callback from Find
+ const std::function<bool(const Status &, const std::vector<StatInfo> &, bool)> handler;
+ //outstanding_requests is incremented once for every GetListing call.
+ std::atomic<uint64_t> outstanding_requests;
+ //Boolean needed to abort all recursion on error or on user command
+ std::atomic<bool> aborted;
+ //Shared variables will need protection with a lock
+ std::mutex lock;
+ FindSharedState(const std::string path_, const std::string name_, const uint32_t maxdepth_,
+ const std::function<bool(const Status &, const std::vector<StatInfo> &, bool)> handler_,
+ uint64_t outstanding_recuests_, bool aborted_)
+ : name(name_),
+ maxdepth(maxdepth_),
+ handler(handler_),
+ outstanding_requests(outstanding_recuests_),
+ aborted(aborted_),
+ lock() {
+ //Constructing the list of sub-directories
+ std::stringstream ss(path_);
+ if(path_.back() != '/'){
+ ss << "/";
+ }
+ for (std::string token; std::getline(ss, token, '/'); ) {
+ dirs.push_back(token);
+ }
+ }
+ };
+
+ struct FindOperationalState {
+ const std::string path;
+ const uint32_t depth;
+ const bool search_path;
+ FindOperationalState(const std::string path_, const uint32_t depth_, const bool search_path_)
+ : path(path_),
+ depth(depth_),
+ search_path(search_path_) {
+ }
+ };
+
+ void FindShim(const Status &stat, const std::vector<StatInfo> & stat_infos,
+ bool directory_has_more, std::shared_ptr<FindOperationalState> current_state, std::shared_ptr<FindSharedState> shared_state);
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
index 27ccb5d..89acac3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
@@ -39,30 +39,6 @@ namespace hdfs {
* NAMENODE OPERATIONS
****************************************************************************/
-uint16_t NameNodeOperations::GetDefaultPermissionMask() {
- return 0755;
-}
-
-Status NameNodeOperations::CheckValidPermissionMask(uint16_t permissions) {
- if (permissions > 01777) {
- std::stringstream errormsg;
- errormsg << "CheckValidPermissionMask: argument 'permissions' is " << std::oct
- << std::showbase << permissions << " (should be between 0 and 01777)";
- return Status::InvalidArgument(errormsg.str().c_str());
- }
- return Status::OK();
-}
-
-Status NameNodeOperations::CheckValidReplication(uint16_t replication) {
- if (replication < 1 || replication > 512) {
- std::stringstream errormsg;
- errormsg << "CheckValidReplication: argument 'replication' is "
- << replication << " (should be between 1 and 512)";
- return Status::InvalidArgument(errormsg.str().c_str());
- }
- return Status::OK();
-}
-
void NameNodeOperations::Connect(const std::string &cluster_name,
const std::vector<ResolvedNamenodeInfo> &servers,
std::function<void(const Status &)> &&handler) {
@@ -170,7 +146,7 @@ void NameNodeOperations::SetReplication(const std::string & path, int16_t replic
handler(Status::InvalidArgument("SetReplication: argument 'path' cannot be empty"));
return;
}
- Status replStatus = CheckValidReplication(replication);
+ Status replStatus = FileSystemImpl::CheckValidReplication(replication);
if (!replStatus.ok()) {
handler(replStatus);
return;
@@ -252,7 +228,8 @@ void NameNodeOperations::GetFileInfo(const std::string & path,
// no fs in the protobuf.
if(resp -> has_fs()){
struct StatInfo stat_info;
- stat_info.path=path;
+ stat_info.path = path;
+ stat_info.full_path = path;
HdfsFileStatusProtoToStatInfo(stat_info, resp->fs());
handler(stat, stat_info);
} else {
@@ -290,7 +267,7 @@ void NameNodeOperations::GetFsStats(
void NameNodeOperations::GetListing(
const std::string & path,
- std::function<void(const Status &, std::shared_ptr<std::vector<StatInfo>> &, bool)> handler,
+ std::function<void(const Status &, const std::vector<StatInfo> &, bool)> handler,
const std::string & start_after) {
using ::hadoop::hdfs::GetListingRequestProto;
using ::hadoop::hdfs::GetListingResponseProto;
@@ -300,8 +277,8 @@ void NameNodeOperations::GetListing(
<< "NameNodeOperations::GetListing(" << FMT_THIS_ADDR << ", path=" << path << ") called");
if (path.empty()) {
- std::shared_ptr<std::vector<StatInfo>> stat_infos;
- handler(Status::InvalidArgument("GetListing: argument 'path' cannot be empty"), stat_infos, false);
+ std::vector<StatInfo> empty;
+ handler(Status::InvalidArgument("GetListing: argument 'path' cannot be empty"), empty, false);
return;
}
@@ -312,31 +289,26 @@ void NameNodeOperations::GetListing(
auto resp = std::make_shared<GetListingResponseProto>();
- namenode_.GetListing(
- &req,
- resp,
- [resp, handler, path](const Status &stat) {
- if (stat.ok()) {
- if(resp -> has_dirlist()){
- std::shared_ptr<std::vector<StatInfo>> stat_infos(new std::vector<StatInfo>);
- for (::hadoop::hdfs::HdfsFileStatusProto const& fs : resp->dirlist().partiallisting()) {
- StatInfo si;
- si.path=fs.path();
- HdfsFileStatusProtoToStatInfo(si, fs);
- stat_infos->push_back(si);
- }
- handler(stat, stat_infos, resp->dirlist().remainingentries() > 0);
- } else {
- std::string errormsg = "No such file or directory: " + path;
- Status statNew = Status::PathNotFound(errormsg.c_str());
- std::shared_ptr<std::vector<StatInfo>> stat_infos;
- handler(statNew, stat_infos, false);
- }
- } else {
- std::shared_ptr<std::vector<StatInfo>> stat_infos;
- handler(stat, stat_infos, false);
+ namenode_.GetListing(&req, resp, [resp, handler, path](const Status &stat) {
+ std::vector<StatInfo> stat_infos;
+ if (stat.ok()) {
+ if(resp -> has_dirlist()){
+ for (::hadoop::hdfs::HdfsFileStatusProto const& fs : resp->dirlist().partiallisting()) {
+ StatInfo si;
+ si.path = fs.path();
+ si.full_path = path + fs.path() + "/";
+ HdfsFileStatusProtoToStatInfo(si, fs);
+ stat_infos.push_back(si);
}
- });
+ handler(stat, stat_infos, resp->dirlist().remainingentries() > 0);
+ } else {
+ std::string errormsg = "No such file or directory: " + path;
+ handler(Status::PathNotFound(errormsg.c_str()), stat_infos, false);
+ }
+ } else {
+ handler(stat, stat_infos, false);
+ }
+ });
}
void NameNodeOperations::Mkdirs(const std::string & path, uint16_t permissions, bool createparent,
@@ -355,7 +327,7 @@ void NameNodeOperations::Mkdirs(const std::string & path, uint16_t permissions,
}
MkdirsRequestProto req;
- Status permStatus = CheckValidPermissionMask(permissions);
+ Status permStatus = FileSystemImpl::CheckValidPermissionMask(permissions);
if (!permStatus.ok()) {
handler(permStatus);
return;
@@ -471,7 +443,7 @@ void NameNodeOperations::SetPermission(const std::string & path,
handler(Status::InvalidArgument("SetPermission: argument 'path' cannot be empty"));
return;
}
- Status permStatus = CheckValidPermissionMask(permissions);
+ Status permStatus = FileSystemImpl::CheckValidPermissionMask(permissions);
if (!permStatus.ok()) {
handler(permStatus);
return;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
index 3afa2e9..60efacc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
@@ -48,12 +48,6 @@ public:
engine_(io_service, options, client_name, user_name, protocol_name, protocol_version),
namenode_(& engine_), options_(options) {}
- static uint16_t GetDefaultPermissionMask();
-
- static Status CheckValidPermissionMask(uint16_t permissions);
-
- static Status CheckValidReplication(uint16_t replication);
-
void Connect(const std::string &cluster_name,
const std::vector<ResolvedNamenodeInfo> &servers,
std::function<void(const Status &)> &&handler);
@@ -77,7 +71,7 @@ public:
// start_after="" for initial call
void GetListing(const std::string & path,
- std::function<void(const Status &, std::shared_ptr<std::vector<StatInfo>>&, bool)> handler,
+ std::function<void(const Status &, const std::vector<StatInfo> &, bool)> handler,
const std::string & start_after = "");
void Mkdirs(const std::string & path, uint16_t permissions, bool createparent,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc
index 4d49728..9534204 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc
@@ -33,21 +33,27 @@ TEST(ConfigurationTest, TestDegenerateInputs) {
/* Completely empty stream */
{
std::stringstream stream;
- optional<Configuration> config = ConfigurationLoader().Load<Configuration>("");
+ ConfigurationLoader config_loader;
+ config_loader.ClearSearchPath();
+ optional<Configuration> config = config_loader.Load<Configuration>("");
EXPECT_FALSE(config && "Empty stream");
}
/* No values */
{
std::string data = "<configuration></configuration>";
- optional<Configuration> config = ConfigurationLoader().Load<Configuration>(data);
+ ConfigurationLoader config_loader;
+ config_loader.ClearSearchPath();
+ optional<Configuration> config = config_loader.Load<Configuration>(data);
EXPECT_TRUE(config && "Blank config");
}
/* Extraneous values */
{
std::string data = "<configuration><spam></spam></configuration>";
- optional<Configuration> config = ConfigurationLoader().Load<Configuration>(data);
+ ConfigurationLoader config_loader;
+ config_loader.ClearSearchPath();
+ optional<Configuration> config = config_loader.Load<Configuration>(data);
EXPECT_TRUE(config && "Extraneous values");
}
}
@@ -57,7 +63,9 @@ TEST(ConfigurationTest, TestBasicOperations) {
{
std::stringstream stream;
simpleConfigStream(stream, "key1", "value1");
- optional<Configuration> config = ConfigurationLoader().Load<Configuration>(stream.str());
+ ConfigurationLoader config_loader;
+ config_loader.ClearSearchPath();
+ optional<Configuration> config = config_loader.Load<Configuration>(stream.str());
EXPECT_TRUE(config && "Parse single value");
EXPECT_EQ("value1", config->GetWithDefault("key1", ""));
}
@@ -74,7 +82,9 @@ TEST(ConfigurationTest, TestBasicOperations) {
{
std::stringstream stream;
simpleConfigStream(stream, "key1", "value1");
- optional<Configuration> config = ConfigurationLoader().Load<Configuration>(stream.str());
+ ConfigurationLoader config_loader;
+ config_loader.ClearSearchPath();
+ optional<Configuration> config = config_loader.Load<Configuration>(stream.str());
EXPECT_TRUE(config && "Parse single value");
EXPECT_EQ("value1", config->GetWithDefault("KEY1", ""));
}
@@ -83,7 +93,9 @@ TEST(ConfigurationTest, TestBasicOperations) {
{
std::stringstream stream;
simpleConfigStream(stream, "key1", "value1");
- optional<Configuration> config = ConfigurationLoader().Load<Configuration>(stream.str());
+ ConfigurationLoader config_loader;
+ config_loader.ClearSearchPath();
+ optional<Configuration> config = config_loader.Load<Configuration>(stream.str());
EXPECT_TRUE(config && "Parse single value");
optional<std::string> value = config->Get("key1");
EXPECT_TRUE((bool)value);
@@ -97,7 +109,9 @@ TEST(ConfigurationTest, TestCompactValues) {
std::stringstream stream;
stream << "<configuration><property name=\"key1\" "
"value=\"value1\"/></configuration>";
- optional<Configuration> config = ConfigurationLoader().Load<Configuration>(stream.str());
+ ConfigurationLoader config_loader;
+ config_loader.ClearSearchPath();
+ optional<Configuration> config = config_loader.Load<Configuration>(stream.str());
EXPECT_TRUE(config && "Compact value parse");
EXPECT_EQ("value1", config->GetWithDefault("key1", ""));
}
@@ -108,7 +122,9 @@ TEST(ConfigurationTest, TestMultipleResources) {
{
std::stringstream stream;
simpleConfigStream(stream, "key1", "value1");
- optional<Configuration> config = ConfigurationLoader().Load<Configuration>(stream.str());
+ ConfigurationLoader config_loader;
+ config_loader.ClearSearchPath();
+ optional<Configuration> config = config_loader.Load<Configuration>(stream.str());
EXPECT_TRUE(config && "Parse first stream");
EXPECT_EQ("value1", config->GetWithDefault("key1", ""));
@@ -129,7 +145,9 @@ TEST(ConfigurationTest, TestStringResource) {
simpleConfigStream(stream, "key1", "value1");
std::string str = stream.str();
- optional<Configuration> config = ConfigurationLoader().Load<Configuration>(stream.str());
+ ConfigurationLoader config_loader;
+ config_loader.ClearSearchPath();
+ optional<Configuration> config = config_loader.Load<Configuration>(stream.str());
EXPECT_TRUE(config && "Parse single value");
EXPECT_EQ("value1", config->GetWithDefault("key1", ""));
}
@@ -171,7 +189,9 @@ TEST(ConfigurationTest, TestFinal) {
std::stringstream stream;
stream << "<configuration><property><name>key1</name><value>value1</"
"value><final>false</final></property></configuration>";
- optional<Configuration> config = ConfigurationLoader().Load<Configuration>(stream.str());
+ ConfigurationLoader config_loader;
+ config_loader.ClearSearchPath();
+ optional<Configuration> config = config_loader.Load<Configuration>(stream.str());
EXPECT_TRUE(config && "Parse first stream");
EXPECT_EQ("value1", config->GetWithDefault("key1", ""));
@@ -187,7 +207,9 @@ TEST(ConfigurationTest, TestFinal) {
std::stringstream stream;
stream << "<configuration><property><name>key1</name><value>value1</"
"value><final>true</final></property></configuration>";
- optional<Configuration> config = ConfigurationLoader().Load<Configuration>(stream.str());
+ ConfigurationLoader config_loader;
+ config_loader.ClearSearchPath();
+ optional<Configuration> config = config_loader.Load<Configuration>(stream.str());
EXPECT_TRUE(config && "Parse first stream");
EXPECT_EQ("value1", config->GetWithDefault("key1", ""));
@@ -203,7 +225,9 @@ TEST(ConfigurationTest, TestFinal) {
std::stringstream stream;
stream << "<configuration><property name=\"key1\" value=\"value1\" "
"final=\"false\"/></configuration>";
- optional<Configuration> config = ConfigurationLoader().Load<Configuration>(stream.str());
+ ConfigurationLoader config_loader;
+ config_loader.ClearSearchPath();
+ optional<Configuration> config = config_loader.Load<Configuration>(stream.str());
EXPECT_TRUE(config && "Parse first stream");
EXPECT_EQ("value1", config->GetWithDefault("key1", ""));
@@ -219,7 +243,9 @@ TEST(ConfigurationTest, TestFinal) {
std::stringstream stream;
stream << "<configuration><property name=\"key1\" value=\"value1\" "
"final=\"true\"/></configuration>";
- optional<Configuration> config = ConfigurationLoader().Load<Configuration>(stream.str());
+ ConfigurationLoader config_loader;
+ config_loader.ClearSearchPath();
+ optional<Configuration> config = config_loader.Load<Configuration>(stream.str());
EXPECT_TRUE(config && "Parse first stream");
EXPECT_EQ("value1", config->GetWithDefault("key1", ""));
@@ -235,7 +261,9 @@ TEST(ConfigurationTest, TestFinal) {
std::stringstream stream;
stream << "<configuration><property><name>key1</name><value>value1</"
"value><final>spam</final></property></configuration>";
- optional<Configuration> config = ConfigurationLoader().Load<Configuration>(stream.str());
+ ConfigurationLoader config_loader;
+ config_loader.ClearSearchPath();
+ optional<Configuration> config = config_loader.Load<Configuration>(stream.str());
EXPECT_TRUE(config && "Parse first stream");
EXPECT_EQ("value1", config->GetWithDefault("key1", ""));
@@ -251,7 +279,9 @@ TEST(ConfigurationTest, TestFinal) {
std::stringstream stream;
stream << "<configuration><property><name>key1</name><value>value1</"
"value><final></final></property></configuration>";
- optional<Configuration> config = ConfigurationLoader().Load<Configuration>(stream.str());
+ ConfigurationLoader config_loader;
+ config_loader.ClearSearchPath();
+ optional<Configuration> config = config_loader.Load<Configuration>(stream.str());
EXPECT_TRUE(config && "Parse first stream");
EXPECT_EQ("value1", config->GetWithDefault("key1", ""));
@@ -271,7 +301,9 @@ TEST(ConfigurationTest, TestFileReads)
TempFile tempFile;
writeSimpleConfig(tempFile.filename, "key1", "value1");
- optional<Configuration> config = ConfigurationLoader().LoadFromFile<Configuration>(tempFile.filename);
+ ConfigurationLoader config_loader;
+ config_loader.ClearSearchPath();
+ optional<Configuration> config = config_loader.LoadFromFile<Configuration>(tempFile.filename);
EXPECT_TRUE(config && "Parse first stream");
EXPECT_EQ("value1", config->GetWithDefault("key1", ""));
}
@@ -298,7 +330,9 @@ TEST(ConfigurationTest, TestFileReads)
{
TempDir tempDir;
- optional<Configuration> config = ConfigurationLoader().LoadFromFile<Configuration>(tempDir.path);
+ ConfigurationLoader config_loader;
+ config_loader.ClearSearchPath();
+ optional<Configuration> config = config_loader.LoadFromFile<Configuration>(tempDir.path);
EXPECT_FALSE(config && "Add directory as file resource");
}
@@ -359,7 +393,9 @@ TEST(ConfigurationTest, TestIntConversions) {
{
std::stringstream stream;
simpleConfigStream(stream, "key1", "1");
- optional<Configuration> config = ConfigurationLoader().Load<Configuration>(stream.str());
+ ConfigurationLoader config_loader;
+ config_loader.ClearSearchPath();
+ optional<Configuration> config = config_loader.Load<Configuration>(stream.str());
EXPECT_TRUE(config && "Parse single value");
optional<int64_t> value = config->GetInt("key1");
EXPECT_TRUE((bool)value);
@@ -398,7 +434,9 @@ TEST(ConfigurationTest, TestDoubleConversions) {
{
std::stringstream stream;
simpleConfigStream(stream, "key1", "1");
- optional<Configuration> config = ConfigurationLoader().Load<Configuration>(stream.str());
+ ConfigurationLoader config_loader;
+ config_loader.ClearSearchPath();
+ optional<Configuration> config = config_loader.Load<Configuration>(stream.str());
EXPECT_TRUE(config && "Parse single value");
optional<double> value = config->GetDouble("key1");
EXPECT_TRUE((bool)value);
@@ -441,7 +479,9 @@ TEST(ConfigurationTest, TestBoolConversions) {
{
std::stringstream stream;
simpleConfigStream(stream, "key1", "true");
- optional<Configuration> config = ConfigurationLoader().Load<Configuration>(stream.str());
+ ConfigurationLoader config_loader;
+ config_loader.ClearSearchPath();
+ optional<Configuration> config = config_loader.Load<Configuration>(stream.str());
EXPECT_TRUE(config && "Parse single value");
optional<bool> value = config->GetBool("key1");
EXPECT_TRUE((bool)value);
@@ -488,7 +528,9 @@ TEST(ConfigurationTest, TestUriConversions) {
{
std::stringstream stream;
simpleConfigStream(stream, "key1", "hdfs:///");
- optional<Configuration> config = ConfigurationLoader().Load<Configuration>(stream.str());
+ ConfigurationLoader config_loader;
+ config_loader.ClearSearchPath();
+ optional<Configuration> config = config_loader.Load<Configuration>(stream.str());
EXPECT_TRUE(config && "Parse single value");
optional<URI> value = config->GetUri("key1");
EXPECT_TRUE((bool)value);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.h
index 669557f..7947ff5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.h
@@ -54,7 +54,9 @@ template <typename... Args>
optional<Configuration> simpleConfig(Args... args) {
std::stringstream stream;
simpleConfigStream(stream, args...);
- optional<Configuration> parse = ConfigurationLoader().Load<Configuration>(stream.str());
+ ConfigurationLoader config_loader;
+ config_loader.ClearSearchPath();
+ optional<Configuration> parse = config_loader.Load<Configuration>(stream.str());
EXPECT_TRUE((bool)parse);
return parse;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6cb5d1/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_configuration_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_configuration_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_configuration_test.cc
index 7e9ca66..360f886 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_configuration_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_configuration_test.cc
@@ -30,7 +30,9 @@ TEST(HdfsConfigurationTest, TestDefaultOptions)
{
// Completely empty stream
{
- HdfsConfiguration empty_config = ConfigurationLoader().New<HdfsConfiguration>();
+ ConfigurationLoader config_loader;
+ config_loader.ClearSearchPath();
+ HdfsConfiguration empty_config = config_loader.NewConfig<HdfsConfiguration>();
Options options = empty_config.GetOptions();
EXPECT_EQ(Options::kDefaultRpcTimeout, options.rpc_timeout);
}
@@ -49,8 +51,9 @@ TEST(HdfsConfigurationTest, TestSetOptions)
HdfsConfiguration::kIpcClientConnectTimeoutKey, 103,
HdfsConfiguration::kHadoopSecurityAuthenticationKey, HdfsConfiguration::kHadoopSecurityAuthentication_kerberos
);
-
- optional<HdfsConfiguration> config = ConfigurationLoader().Load<HdfsConfiguration>(stream.str());
+ ConfigurationLoader config_loader;
+ config_loader.ClearSearchPath();
+ optional<HdfsConfiguration> config = config_loader.Load<HdfsConfiguration>(stream.str());
EXPECT_TRUE(config && "Read stream");
Options options = config->GetOptions();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/52] [abbrv] hadoop git commit: HDFS-12013: libhdfs++: read with
offset at EOF should return 0 bytes instead of error. Contributed by Xiaowei
Zhu
Posted by jh...@apache.org.
HDFS-12013: libhdfs++: read with offset at EOF should return 0 bytes instead of error. Contributed by Xiaowei Zhu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3c743b47
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3c743b47
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3c743b47
Branch: refs/heads/HDFS-8707
Commit: 3c743b475e6ec431a5849ebd467b077e41e89987
Parents: c5e7a69
Author: James Clampffer <ja...@hp.com>
Authored: Fri Jul 7 14:04:42 2017 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../src/main/native/libhdfspp/lib/fs/filehandle.cc | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c743b47/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
index eea7ac9..2087d53 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
@@ -181,7 +181,10 @@ void FileHandleImpl::AsyncPreadSome(
return;
}
- if(offset >= file_info_->file_length_){
+ if(offset == file_info_->file_length_) {
+ handler(Status::OK(), "", 0);
+ return;
+ } else if(offset > file_info_->file_length_){
handler(Status::InvalidOffset("AsyncPreadSome: trying to begin a read past the EOF"), "", 0);
return;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/52] [abbrv] hadoop git commit: HDFS-11106: libhdfs++: Some
refactoring to better organize files (part 2). Contributed by James
Clampffer.
Posted by jh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e53da2d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h
new file mode 100644
index 0000000..12f7f0e
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h
@@ -0,0 +1,445 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef LIB_RPC_RPC_CONNECTION_IMPL_H_
+#define LIB_RPC_RPC_CONNECTION_IMPL_H_
+
+#include "rpc_connection.h"
+#include "rpc_engine.h"
+#include "request.h"
+
+#include "common/auth_info.h"
+#include "common/logging.h"
+#include "common/util.h"
+#include "common/libhdfs_events_impl.h"
+
+#include <asio/connect.hpp>
+#include <asio/read.hpp>
+#include <asio/write.hpp>
+
+#include <system_error>
+
+namespace hdfs {
+
+template <class Socket>
+class RpcConnectionImpl : public RpcConnection {
+public:
+ MEMCHECKED_CLASS(RpcConnectionImpl);
+
+ RpcConnectionImpl(RpcEngine *engine);
+ virtual ~RpcConnectionImpl() override;
+
+ virtual void Connect(const std::vector<::asio::ip::tcp::endpoint> &server,
+ const AuthInfo & auth_info,
+ RpcCallback &handler);
+ virtual void ConnectAndFlush(
+ const std::vector<::asio::ip::tcp::endpoint> &server) override;
+ virtual void SendHandshake(RpcCallback &handler) override;
+ virtual void SendContext(RpcCallback &handler) override;
+ virtual void Disconnect() override;
+ virtual void OnSendCompleted(const ::asio::error_code &ec,
+ size_t transferred) override;
+ virtual void OnRecvCompleted(const ::asio::error_code &ec,
+ size_t transferred) override;
+ virtual void FlushPendingRequests() override;
+
+
+ Socket &TEST_get_mutable_socket() { return socket_; }
+
+ void TEST_set_connected(bool connected) { connected_ = connected ? kConnected : kNotYetConnected; }
+
+ private:
+ const Options options_;
+ ::asio::ip::tcp::endpoint current_endpoint_;
+ std::vector<::asio::ip::tcp::endpoint> additional_endpoints_;
+ Socket socket_;
+ ::asio::deadline_timer connect_timer_;
+
+ void ConnectComplete(const ::asio::error_code &ec, const ::asio::ip::tcp::endpoint &remote);
+};
+
+template <class Socket>
+RpcConnectionImpl<Socket>::RpcConnectionImpl(RpcEngine *engine)
+ : RpcConnection(engine),
+ options_(engine->options()),
+ socket_(engine->io_service()),
+ connect_timer_(engine->io_service())
+{
+ LOG_TRACE(kRPC, << "RpcConnectionImpl::RpcConnectionImpl called &" << (void*)this);
+}
+
+template <class Socket>
+RpcConnectionImpl<Socket>::~RpcConnectionImpl() {
+ LOG_DEBUG(kRPC, << "RpcConnectionImpl::~RpcConnectionImpl called &" << (void*)this);
+
+ if (pending_requests_.size() > 0)
+ LOG_WARN(kRPC, << "RpcConnectionImpl::~RpcConnectionImpl called with items in the pending queue");
+ if (requests_on_fly_.size() > 0)
+ LOG_WARN(kRPC, << "RpcConnectionImpl::~RpcConnectionImpl called with items in the requests_on_fly queue");
+}
+
+template <class Socket>
+void RpcConnectionImpl<Socket>::Connect(
+ const std::vector<::asio::ip::tcp::endpoint> &server,
+ const AuthInfo & auth_info,
+ RpcCallback &handler) {
+ LOG_TRACE(kRPC, << "RpcConnectionImpl::Connect called");
+
+ this->auth_info_ = auth_info;
+
+ auto connectionSuccessfulReq = std::make_shared<Request>(
+ engine_, [handler](::google::protobuf::io::CodedInputStream *is,
+ const Status &status) {
+ (void)is;
+ handler(status);
+ });
+ pending_requests_.push_back(connectionSuccessfulReq);
+ this->ConnectAndFlush(server); // need "this" so compiler can infer type of CAF
+}
+
+template <class Socket>
+void RpcConnectionImpl<Socket>::ConnectAndFlush(
+ const std::vector<::asio::ip::tcp::endpoint> &server) {
+
+ LOG_INFO(kRPC, << "ConnectAndFlush called");
+ std::lock_guard<std::mutex> state_lock(connection_state_lock_);
+
+ if (server.empty()) {
+ Status s = Status::InvalidArgument("No endpoints provided");
+ CommsError(s);
+ return;
+ }
+
+ if (connected_ == kConnected) {
+ FlushPendingRequests();
+ return;
+ }
+ if (connected_ != kNotYetConnected) {
+ LOG_WARN(kRPC, << "RpcConnectionImpl::ConnectAndFlush called while connected=" << ToString(connected_));
+ return;
+ }
+ connected_ = kConnecting;
+
+ // Take the first endpoint, but remember the alternatives for later
+ additional_endpoints_ = server;
+ ::asio::ip::tcp::endpoint first_endpoint = additional_endpoints_.front();
+ additional_endpoints_.erase(additional_endpoints_.begin());
+ current_endpoint_ = first_endpoint;
+
+ auto shared_this = shared_from_this();
+ socket_.async_connect(first_endpoint, [shared_this, this, first_endpoint](const ::asio::error_code &ec) {
+ ConnectComplete(ec, first_endpoint);
+ });
+
+ // Prompt the timer to timeout
+ auto weak_this = std::weak_ptr<RpcConnection>(shared_this);
+ connect_timer_.expires_from_now(
+ std::chrono::milliseconds(options_.rpc_connect_timeout));
+ connect_timer_.async_wait([shared_this, this, first_endpoint](const ::asio::error_code &ec) {
+ if (ec)
+ ConnectComplete(ec, first_endpoint);
+ else
+ ConnectComplete(make_error_code(asio::error::host_unreachable), first_endpoint);
+ });
+}
+
+template <class Socket>
+void RpcConnectionImpl<Socket>::ConnectComplete(const ::asio::error_code &ec, const ::asio::ip::tcp::endpoint & remote) {
+ auto shared_this = RpcConnectionImpl<Socket>::shared_from_this();
+ std::lock_guard<std::mutex> state_lock(connection_state_lock_);
+ connect_timer_.cancel();
+
+ LOG_TRACE(kRPC, << "RpcConnectionImpl::ConnectComplete called");
+
+ // Could be an old async connect returning a result after we've moved on
+ if (remote != current_endpoint_) {
+ LOG_DEBUG(kRPC, << "Got ConnectComplete for " << remote << " but current_endpoint_ is " << current_endpoint_);
+ return;
+ }
+ if (connected_ != kConnecting) {
+ LOG_DEBUG(kRPC, << "Got ConnectComplete but current state is " << connected_);;
+ return;
+ }
+
+ Status status = ToStatus(ec);
+ if(event_handlers_) {
+ event_response event_resp = event_handlers_->call(FS_NN_CONNECT_EVENT, cluster_name_.c_str(), 0);
+#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
+ if (event_resp.response() == event_response::kTest_Error) {
+ status = event_resp.status();
+ }
+#endif
+ }
+
+ if (status.ok()) {
+ StartReading();
+ SendHandshake([shared_this, this](const Status & s) {
+ HandshakeComplete(s);
+ });
+ } else {
+ LOG_DEBUG(kRPC, << "Rpc connection failed; err=" << status.ToString());;
+ std::string err = SafeDisconnect(get_asio_socket_ptr(&socket_));
+ if(!err.empty()) {
+ LOG_INFO(kRPC, << "Rpc connection failed to connect to endpoint, error closing connection: " << err);
+ }
+
+ if (!additional_endpoints_.empty()) {
+ // If we have additional endpoints, keep trying until we either run out or
+ // hit one
+ ::asio::ip::tcp::endpoint next_endpoint = additional_endpoints_.front();
+ additional_endpoints_.erase(additional_endpoints_.begin());
+ current_endpoint_ = next_endpoint;
+
+ socket_.async_connect(next_endpoint, [shared_this, this, next_endpoint](const ::asio::error_code &ec) {
+ ConnectComplete(ec, next_endpoint);
+ });
+ connect_timer_.expires_from_now(
+ std::chrono::milliseconds(options_.rpc_connect_timeout));
+ connect_timer_.async_wait([shared_this, this, next_endpoint](const ::asio::error_code &ec) {
+ if (ec)
+ ConnectComplete(ec, next_endpoint);
+ else
+ ConnectComplete(make_error_code(asio::error::host_unreachable), next_endpoint);
+ });
+ } else {
+ CommsError(status);
+ }
+ }
+}
+
+template <class Socket>
+void RpcConnectionImpl<Socket>::SendHandshake(RpcCallback &handler) {
+ assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
+
+ LOG_TRACE(kRPC, << "RpcConnectionImpl::SendHandshake called");
+ connected_ = kHandshaking;
+
+ auto shared_this = shared_from_this();
+ auto handshake_packet = PrepareHandshakePacket();
+ ::asio::async_write(socket_, asio::buffer(*handshake_packet),
+ [handshake_packet, handler, shared_this, this](
+ const ::asio::error_code &ec, size_t) {
+ Status status = ToStatus(ec);
+ handler(status);
+ });
+}
+
+template <class Socket>
+void RpcConnectionImpl<Socket>::SendContext(RpcCallback &handler) {
+ assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
+
+ LOG_TRACE(kRPC, << "RpcConnectionImpl::SendContext called");
+
+ auto shared_this = shared_from_this();
+ auto context_packet = PrepareContextPacket();
+ ::asio::async_write(socket_, asio::buffer(*context_packet),
+ [context_packet, handler, shared_this, this](
+ const ::asio::error_code &ec, size_t) {
+ Status status = ToStatus(ec);
+ handler(status);
+ });
+}
+
+template <class Socket>
+void RpcConnectionImpl<Socket>::OnSendCompleted(const ::asio::error_code &ec,
+ size_t) {
+ using std::placeholders::_1;
+ using std::placeholders::_2;
+ std::lock_guard<std::mutex> state_lock(connection_state_lock_);
+
+ LOG_TRACE(kRPC, << "RpcConnectionImpl::OnSendCompleted called");
+
+ request_over_the_wire_.reset();
+ if (ec) {
+ LOG_WARN(kRPC, << "Network error during RPC write: " << ec.message());
+ CommsError(ToStatus(ec));
+ return;
+ }
+
+ FlushPendingRequests();
+}
+
+template <class Socket>
+void RpcConnectionImpl<Socket>::FlushPendingRequests() {
+ using namespace ::std::placeholders;
+
+ // Lock should be held
+ assert(lock_held(connection_state_lock_));
+
+ LOG_TRACE(kRPC, << "RpcConnectionImpl::FlushPendingRequests called");
+
+ // Don't send if we don't need to
+ if (request_over_the_wire_) {
+ return;
+ }
+
+ std::shared_ptr<Request> req;
+ switch (connected_) {
+ case kNotYetConnected:
+ return;
+ case kConnecting:
+ return;
+ case kHandshaking:
+ return;
+ case kAuthenticating:
+ if (auth_requests_.empty()) {
+ return;
+ }
+ req = auth_requests_.front();
+ auth_requests_.erase(auth_requests_.begin());
+ break;
+ case kConnected:
+ if (pending_requests_.empty()) {
+ return;
+ }
+ req = pending_requests_.front();
+ pending_requests_.erase(pending_requests_.begin());
+ break;
+ case kDisconnected:
+ LOG_DEBUG(kRPC, << "RpcConnectionImpl::FlushPendingRequests attempted to flush a " << ToString(connected_) << " connection");
+ return;
+ default:
+ LOG_DEBUG(kRPC, << "RpcConnectionImpl::FlushPendingRequests invalid state: " << ToString(connected_));
+ return;
+ }
+
+ std::shared_ptr<RpcConnection> shared_this = shared_from_this();
+ auto weak_this = std::weak_ptr<RpcConnection>(shared_this);
+ auto weak_req = std::weak_ptr<Request>(req);
+
+ std::shared_ptr<std::string> payload = std::make_shared<std::string>();
+ req->GetPacket(payload.get());
+ if (!payload->empty()) {
+ assert(requests_on_fly_.find(req->call_id()) == requests_on_fly_.end());
+ requests_on_fly_[req->call_id()] = req;
+ request_over_the_wire_ = req;
+
+ req->timer().expires_from_now(
+ std::chrono::milliseconds(options_.rpc_timeout));
+ req->timer().async_wait([weak_this, weak_req, this](const ::asio::error_code &ec) {
+ auto timeout_this = weak_this.lock();
+ auto timeout_req = weak_req.lock();
+ if (timeout_this && timeout_req)
+ this->HandleRpcTimeout(timeout_req, ec);
+ });
+
+ asio::async_write(socket_, asio::buffer(*payload),
+ [shared_this, this, payload](const ::asio::error_code &ec,
+ size_t size) {
+ OnSendCompleted(ec, size);
+ });
+ } else { // Nothing to send for this request, inform the handler immediately
+ io_service().post(
+ // Never hold locks when calling a callback
+ [req]() { req->OnResponseArrived(nullptr, Status::OK()); }
+ );
+
+ // Reschedule to flush the next one
+ AsyncFlushPendingRequests();
+ }
+}
+
+
+template <class Socket>
+void RpcConnectionImpl<Socket>::OnRecvCompleted(const ::asio::error_code &original_ec,
+ size_t) {
+ using std::placeholders::_1;
+ using std::placeholders::_2;
+ std::lock_guard<std::mutex> state_lock(connection_state_lock_);
+
+ ::asio::error_code my_ec(original_ec);
+
+ LOG_TRACE(kRPC, << "RpcConnectionImpl::OnRecvCompleted called");
+
+ std::shared_ptr<RpcConnection> shared_this = shared_from_this();
+
+ if(event_handlers_) {
+ event_response event_resp = event_handlers_->call(FS_NN_READ_EVENT, cluster_name_.c_str(), 0);
+#ifndef LIBHDFSPP_SIMULATE_ERROR_DISABLED
+ if (event_resp.response() == event_response::kTest_Error) {
+ my_ec = std::make_error_code(std::errc::network_down);
+ }
+#endif
+ }
+
+ switch (my_ec.value()) {
+ case 0:
+ // No errors
+ break;
+ case asio::error::operation_aborted:
+ // The event loop has been shut down. Ignore the error.
+ return;
+ default:
+ LOG_WARN(kRPC, << "Network error during RPC read: " << my_ec.message());
+ CommsError(ToStatus(my_ec));
+ return;
+ }
+
+ if (!current_response_state_) { /* start a new one */
+ current_response_state_ = std::make_shared<Response>();
+ }
+
+ if (current_response_state_->state_ == Response::kReadLength) {
+ current_response_state_->state_ = Response::kReadContent;
+ auto buf = ::asio::buffer(reinterpret_cast<char *>(¤t_response_state_->length_),
+ sizeof(current_response_state_->length_));
+ asio::async_read(
+ socket_, buf,
+ [shared_this, this](const ::asio::error_code &ec, size_t size) {
+ OnRecvCompleted(ec, size);
+ });
+ } else if (current_response_state_->state_ == Response::kReadContent) {
+ current_response_state_->state_ = Response::kParseResponse;
+ current_response_state_->length_ = ntohl(current_response_state_->length_);
+ current_response_state_->data_.resize(current_response_state_->length_);
+ asio::async_read(
+ socket_, ::asio::buffer(current_response_state_->data_),
+ [shared_this, this](const ::asio::error_code &ec, size_t size) {
+ OnRecvCompleted(ec, size);
+ });
+ } else if (current_response_state_->state_ == Response::kParseResponse) {
+ // Check return status from the RPC response. We may have received a msg
+ // indicating a server side error.
+
+ Status stat = HandleRpcResponse(current_response_state_);
+
+ if(stat.get_server_exception_type() == Status::kStandbyException) {
+ // May need to bail out, connect to new NN, and restart loop
+ LOG_INFO(kRPC, << "Communicating with standby NN, attempting to reconnect");
+ }
+
+ current_response_state_ = nullptr;
+ StartReading();
+ }
+}
+
+template <class Socket>
+void RpcConnectionImpl<Socket>::Disconnect() {
+ assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
+
+ LOG_INFO(kRPC, << "RpcConnectionImpl::Disconnect called");
+
+ request_over_the_wire_.reset();
+ if (connected_ == kConnecting || connected_ == kHandshaking || connected_ == kAuthenticating || connected_ == kConnected) {
+ // Don't print out errors, we were expecting a disconnect here
+ SafeDisconnect(get_asio_socket_ptr(&socket_));
+ }
+ connected_ = kDisconnected;
+}
+}
+
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e53da2d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
index 651225a..22c0e74 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
@@ -16,13 +16,12 @@
* limitations under the License.
*/
#include "rpc_engine.h"
-#include "rpc_connection.h"
+#include "rpc_connection_impl.h"
#include "common/util.h"
#include "common/logging.h"
#include "common/namenode_info.h"
#include "optional.hpp"
-#include <future>
#include <algorithm>
namespace hdfs {
@@ -30,114 +29,6 @@ namespace hdfs {
template <class T>
using optional = std::experimental::optional<T>;
-HANamenodeTracker::HANamenodeTracker(const std::vector<ResolvedNamenodeInfo> &servers,
- ::asio::io_service *ioservice,
- std::shared_ptr<LibhdfsEvents> event_handlers)
- : enabled_(false), resolved_(false),
- ioservice_(ioservice), event_handlers_(event_handlers)
-{
- LOG_TRACE(kRPC, << "HANamenodeTracker got the following nodes");
- for(unsigned int i=0;i<servers.size();i++)
- LOG_TRACE(kRPC, << servers[i].str());
-
- if(servers.size() >= 2) {
- LOG_TRACE(kRPC, << "Creating HA namenode tracker");
- if(servers.size() > 2) {
- LOG_WARN(kRPC, << "Nameservice declares more than two nodes. Some won't be used.");
- }
-
- active_info_ = servers[0];
- standby_info_ = servers[1];
- LOG_INFO(kRPC, << "Active namenode url = " << active_info_.uri.str());
- LOG_INFO(kRPC, << "Standby namenode url = " << standby_info_.uri.str());
-
- enabled_ = true;
- if(!active_info_.endpoints.empty() || !standby_info_.endpoints.empty()) {
- resolved_ = true;
- }
- }
-}
-
-
-HANamenodeTracker::~HANamenodeTracker() { }
-
-
-static std::string format_endpoints(const std::vector<::asio::ip::tcp::endpoint> &pts) {
- std::stringstream ss;
- for(unsigned int i=0; i<pts.size(); i++)
- if(i == pts.size() - 1)
- ss << pts[i];
- else
- ss << pts[i] << ", ";
- return ss.str();
-}
-
-// Pass in endpoint from current connection, this will do a reverse lookup
-// and return the info for the standby node. It will also swap its state internally.
-ResolvedNamenodeInfo HANamenodeTracker::GetFailoverAndUpdate(::asio::ip::tcp::endpoint current_endpoint) {
- LOG_TRACE(kRPC, << "Swapping from endpoint " << current_endpoint);
- mutex_guard swap_lock(swap_lock_);
-
- ResolvedNamenodeInfo failover_node;
-
- // Connected to standby, switch standby to active
- if(IsCurrentActive_locked(current_endpoint)) {
- std::swap(active_info_, standby_info_);
- if(event_handlers_)
- event_handlers_->call(FS_NN_FAILOVER_EVENT, active_info_.nameservice.c_str(),
- reinterpret_cast<int64_t>(active_info_.uri.str().c_str()));
- failover_node = active_info_;
- } else if(IsCurrentStandby_locked(current_endpoint)) {
- // Connected to standby
- if(event_handlers_)
- event_handlers_->call(FS_NN_FAILOVER_EVENT, active_info_.nameservice.c_str(),
- reinterpret_cast<int64_t>(active_info_.uri.str().c_str()));
- failover_node = active_info_;
- } else {
- // Invalid state, throw for testing
- std::string ep1 = format_endpoints(active_info_.endpoints);
- std::string ep2 = format_endpoints(standby_info_.endpoints);
-
- std::stringstream msg;
- msg << "Looked for " << current_endpoint << " in\n";
- msg << ep1 << " and\n";
- msg << ep2 << std::endl;
-
- LOG_ERROR(kRPC, << "Unable to find RPC connection in config " << msg.str() << ". Bailing out.");
- throw std::runtime_error(msg.str());
- }
-
- if(failover_node.endpoints.empty()) {
- LOG_WARN(kRPC, << "No endpoints for node " << failover_node.uri.str() << " attempting to resolve again");
- if(!ResolveInPlace(ioservice_, failover_node)) {
- LOG_ERROR(kRPC, << "Fallback endpoint resolution for node " << failover_node.uri.str()
- << "failed. Please make sure your configuration is up to date.");
- }
- }
- return failover_node;
-}
-
-bool HANamenodeTracker::IsCurrentActive_locked(const ::asio::ip::tcp::endpoint &ep) const {
- for(unsigned int i=0;i<active_info_.endpoints.size();i++) {
- if(ep.address() == active_info_.endpoints[i].address()) {
- if(ep.port() != active_info_.endpoints[i].port())
- LOG_WARN(kRPC, << "Port mismatch: " << ep << " vs " << active_info_.endpoints[i] << " trying anyway..");
- return true;
- }
- }
- return false;
-}
-
-bool HANamenodeTracker::IsCurrentStandby_locked(const ::asio::ip::tcp::endpoint &ep) const {
- for(unsigned int i=0;i<standby_info_.endpoints.size();i++) {
- if(ep.address() == standby_info_.endpoints[i].address()) {
- if(ep.port() != standby_info_.endpoints[i].port())
- LOG_WARN(kRPC, << "Port mismatch: " << ep << " vs " << standby_info_.endpoints[i] << " trying anyway..");
- return true;
- }
- }
- return false;
-}
RpcEngine::RpcEngine(::asio::io_service *io_service, const Options &options,
const std::string &client_name, const std::string &user_name,
@@ -276,19 +167,6 @@ void RpcEngine::AsyncRpc(
conn_->AsyncRpc(method_name, req, resp, handler);
}
-Status RpcEngine::Rpc(
- const std::string &method_name, const ::google::protobuf::MessageLite *req,
- const std::shared_ptr<::google::protobuf::MessageLite> &resp) {
-
- LOG_TRACE(kRPC, << "RpcEngine::Rpc called");
-
- auto stat = std::make_shared<std::promise<Status>>();
- std::future<Status> future(stat->get_future());
- AsyncRpc(method_name, req, resp,
- [stat](const Status &status) { stat->set_value(status); });
- return future.get();
-}
-
std::shared_ptr<RpcConnection> RpcEngine::NewConnection()
{
LOG_DEBUG(kRPC, << "RpcEngine::NewConnection called");
@@ -304,7 +182,6 @@ std::shared_ptr<RpcConnection> RpcEngine::InitializeConnection()
return result;
}
-
void RpcEngine::AsyncRpcCommsError(
const Status &status,
std::shared_ptr<RpcConnection> failedConnection,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e53da2d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
index b4aef00..dc4054e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
@@ -25,25 +25,19 @@
#include "common/retry_policy.h"
#include "common/libhdfs_events_impl.h"
#include "common/util.h"
-#include "common/continuation/asio.h"
-#include "common/logging.h"
#include "common/new_delete.h"
#include "common/namenode_info.h"
+#include "namenode_tracker.h"
#include <google/protobuf/message_lite.h>
-#include <google/protobuf/io/coded_stream.h>
-#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
#include <asio/ip/tcp.hpp>
#include <asio/deadline_timer.hpp>
#include <atomic>
#include <memory>
-#include <unordered_map>
#include <vector>
-#include <deque>
#include <mutex>
-#include <future>
namespace hdfs {
@@ -64,193 +58,8 @@ typedef const std::function<void(const Status &)> RpcCallback;
class LockFreeRpcEngine;
class RpcConnection;
class SaslProtocol;
-
-/*
- * Internal bookkeeping for an outstanding request from the consumer.
- *
- * Threading model: not thread-safe; should only be accessed from a single
- * thread at a time
- */
-class Request {
- public:
- MEMCHECKED_CLASS(Request)
- typedef std::function<void(::google::protobuf::io::CodedInputStream *is,
- const Status &status)> Handler;
-
- Request(LockFreeRpcEngine *engine, const std::string &method_name, int call_id,
- const std::string &request, Handler &&callback);
- Request(LockFreeRpcEngine *engine, const std::string &method_name, int call_id,
- const ::google::protobuf::MessageLite *request, Handler &&callback);
-
- // Null request (with no actual message) used to track the state of an
- // initial Connect call
- Request(LockFreeRpcEngine *engine, Handler &&handler);
-
- int call_id() const { return call_id_; }
- std::string method_name() const { return method_name_; }
- ::asio::deadline_timer &timer() { return timer_; }
- int IncrementRetryCount() { return retry_count_++; }
- int IncrementFailoverCount();
- void GetPacket(std::string *res) const;
- void OnResponseArrived(::google::protobuf::io::CodedInputStream *is,
- const Status &status);
-
- int get_failover_count() {return failover_count_;}
-
- std::string GetDebugString() const;
-
- private:
- LockFreeRpcEngine *const engine_;
- const std::string method_name_;
- const int call_id_;
-
- ::asio::deadline_timer timer_;
- std::string payload_;
- const Handler handler_;
-
- int retry_count_;
- int failover_count_;
-};
-
-/*
- * Encapsulates a persistent connection to the NameNode, and the sending of
- * RPC requests and evaluating their responses.
- *
- * Can have multiple RPC requests in-flight simultaneously, but they are
- * evaluated in-order on the server side in a blocking manner.
- *
- * Threading model: public interface is thread-safe
- * All handlers passed in to method calls will be called from an asio thread,
- * and will not be holding any internal RpcConnection locks.
- */
-class RpcConnection : public std::enable_shared_from_this<RpcConnection> {
- public:
- MEMCHECKED_CLASS(RpcConnection)
- RpcConnection(LockFreeRpcEngine *engine);
- virtual ~RpcConnection();
-
- // Note that a single server can have multiple endpoints - especially both
- // an ipv4 and ipv6 endpoint
- virtual void Connect(const std::vector<::asio::ip::tcp::endpoint> &server,
- const AuthInfo & auth_info,
- RpcCallback &handler) = 0;
- virtual void ConnectAndFlush(const std::vector<::asio::ip::tcp::endpoint> &server) = 0;
- virtual void Disconnect() = 0;
-
- void StartReading();
- void AsyncRpc(const std::string &method_name,
- const ::google::protobuf::MessageLite *req,
- std::shared_ptr<::google::protobuf::MessageLite> resp,
- const RpcCallback &handler);
-
- void AsyncRpc(const std::vector<std::shared_ptr<Request> > & requests);
-
- // Enqueue requests before the connection is connected. Will be flushed
- // on connect
- void PreEnqueueRequests(std::vector<std::shared_ptr<Request>> requests);
-
- // Put requests at the front of the current request queue
- void PrependRequests_locked(std::vector<std::shared_ptr<Request>> requests);
-
- void SetEventHandlers(std::shared_ptr<LibhdfsEvents> event_handlers);
- void SetClusterName(std::string cluster_name);
-
- LockFreeRpcEngine *engine() { return engine_; }
- ::asio::io_service &io_service();
-
- protected:
- struct Response {
- enum ResponseState {
- kReadLength,
- kReadContent,
- kParseResponse,
- } state_;
- unsigned length_;
- std::vector<char> data_;
-
- std::unique_ptr<::google::protobuf::io::ArrayInputStream> ar;
- std::unique_ptr<::google::protobuf::io::CodedInputStream> in;
-
- Response() : state_(kReadLength), length_(0) {}
- };
-
-
- // Initial handshaking protocol: connect->handshake-->(auth)?-->context->connected
- virtual void SendHandshake(RpcCallback &handler) = 0;
- void HandshakeComplete(const Status &s);
- void AuthComplete(const Status &s, const AuthInfo & new_auth_info);
- void AuthComplete_locked(const Status &s, const AuthInfo & new_auth_info);
- virtual void SendContext(RpcCallback &handler) = 0;
- void ContextComplete(const Status &s);
-
-
- virtual void OnSendCompleted(const ::asio::error_code &ec,
- size_t transferred) = 0;
- virtual void OnRecvCompleted(const ::asio::error_code &ec,
- size_t transferred) = 0;
- virtual void FlushPendingRequests()=0; // Synchronously write the next request
-
- void AsyncRpc_locked(
- const std::string &method_name,
- const ::google::protobuf::MessageLite *req,
- std::shared_ptr<::google::protobuf::MessageLite> resp,
- const RpcCallback &handler);
- void SendRpcRequests(const std::vector<std::shared_ptr<Request> > & requests);
- void AsyncFlushPendingRequests(); // Queue requests to be flushed at a later time
-
-
-
- std::shared_ptr<std::string> PrepareHandshakePacket();
- std::shared_ptr<std::string> PrepareContextPacket();
- static std::string SerializeRpcRequest(
- const std::string &method_name,
- const ::google::protobuf::MessageLite *req);
- Status HandleRpcResponse(std::shared_ptr<Response> response);
- void HandleRpcTimeout(std::shared_ptr<Request> req,
- const ::asio::error_code &ec);
- void CommsError(const Status &status);
-
- void ClearAndDisconnect(const ::asio::error_code &ec);
- std::shared_ptr<Request> RemoveFromRunningQueue(int call_id);
-
- LockFreeRpcEngine *const engine_;
- std::shared_ptr<Response> current_response_state_;
- AuthInfo auth_info_;
-
- // Connection can have deferred connection, especially when we're pausing
- // during retry
- enum ConnectedState {
- kNotYetConnected,
- kConnecting,
- kHandshaking,
- kAuthenticating,
- kConnected,
- kDisconnected
- };
- static std::string ToString(ConnectedState connected);
- ConnectedState connected_;
-
- // State machine for performing a SASL handshake
- std::shared_ptr<SaslProtocol> sasl_protocol_;
- // The request being sent over the wire; will also be in requests_on_fly_
- std::shared_ptr<Request> request_over_the_wire_;
- // Requests to be sent over the wire
- std::deque<std::shared_ptr<Request>> pending_requests_;
- // Requests to be sent over the wire during authentication; not retried if
- // there is a connection error
- std::deque<std::shared_ptr<Request>> auth_requests_;
- // Requests that are waiting for responses
- typedef std::unordered_map<int, std::shared_ptr<Request>> RequestOnFlyMap;
- RequestOnFlyMap requests_on_fly_;
- std::shared_ptr<LibhdfsEvents> event_handlers_;
- std::string cluster_name_;
-
- // Lock for mutable parts of this class that need to be thread safe
- std::mutex connection_state_lock_;
-
- friend class SaslProtocol;
-};
-
+class RpcConnection;
+class Request;
/*
* These methods of the RpcEngine will never acquire locks, and are safe for
@@ -259,6 +68,7 @@ class RpcConnection : public std::enable_shared_from_this<RpcConnection> {
class LockFreeRpcEngine {
public:
MEMCHECKED_CLASS(LockFreeRpcEngine)
+
/* Enqueues a CommsError without acquiring a lock*/
virtual void AsyncRpcCommsError(const Status &status,
std::shared_ptr<RpcConnection> failedConnection,
@@ -279,54 +89,6 @@ public:
/*
- * Tracker gives the RpcEngine a quick way to use an endpoint that just
- * failed in order to lookup a set of endpoints for a failover node.
- *
- * Note: For now this only deals with 2 NameNodes, but that's the default
- * anyway.
- */
-class HANamenodeTracker {
- public:
- HANamenodeTracker(const std::vector<ResolvedNamenodeInfo> &servers,
- ::asio::io_service *ioservice,
- std::shared_ptr<LibhdfsEvents> event_handlers_);
-
- virtual ~HANamenodeTracker();
-
- bool is_enabled() const { return enabled_; }
- bool is_resolved() const { return resolved_; }
-
- // Get node opposite of the current one if possible (swaps active/standby)
- // Note: This will always mutate internal state. Use IsCurrentActive/Standby to
- // get info without changing state
- ResolvedNamenodeInfo GetFailoverAndUpdate(::asio::ip::tcp::endpoint current_endpoint);
-
- bool IsCurrentActive_locked(const ::asio::ip::tcp::endpoint &ep) const;
- bool IsCurrentStandby_locked(const ::asio::ip::tcp::endpoint &ep) const;
-
- private:
- // If HA should be enabled, according to our options and runtime info like # nodes provided
- bool enabled_;
- // If we were able to resolve at least 1 HA namenode
- bool resolved_;
-
- // Keep service in case a second round of DNS lookup is required
- ::asio::io_service *ioservice_;
-
- // Event handlers, for now this is the simplest place to catch all failover events
- // and push info out to client application. Possibly move into RPCEngine.
- std::shared_ptr<LibhdfsEvents> event_handlers_;
-
- // Only support 1 active and 1 standby for now.
- ResolvedNamenodeInfo active_info_;
- ResolvedNamenodeInfo standby_info_;
-
- // Aquire when switching from active-standby
- std::mutex swap_lock_;
-};
-
-
-/*
* An engine for reliable communication with a NameNode. Handles connection,
* retry, and (someday) failover of the requested messages.
*
@@ -360,10 +122,6 @@ class RpcEngine : public LockFreeRpcEngine {
const std::shared_ptr<::google::protobuf::MessageLite> &resp,
const std::function<void(const Status &)> &handler);
- Status Rpc(const std::string &method_name,
- const ::google::protobuf::MessageLite *req,
- const std::shared_ptr<::google::protobuf::MessageLite> &resp);
-
void Shutdown();
/* Enqueues a CommsError without acquiring a lock*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e53da2d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
index b35a2f9..83d4f88 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
@@ -17,6 +17,7 @@
*/
#include "rpc_engine.h"
+#include "rpc_connection.h"
#include "common/logging.h"
#include "sasl_engine.h"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e53da2d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
index 08218f6..66a5f1b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
@@ -19,7 +19,7 @@
#include "mock_connection.h"
#include "test.pb.h"
#include "RpcHeader.pb.h"
-#include "rpc/rpc_connection.h"
+#include "rpc/rpc_connection_impl.h"
#include "common/namenode_info.h"
#include <google/protobuf/io/coded_stream.h>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/52] [abbrv] hadoop git commit: HDFS-11518: libhdfs++: Add a build
option to skip building examples, tests,
and tools. Contributed by Anatoli Schein.
Posted by jh...@apache.org.
HDFS-11518: libhdfs++: Add a build option to skip building examples, tests, and tools. Contributed by Anatoli Schein.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1c9b1746
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1c9b1746
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1c9b1746
Branch: refs/heads/HDFS-8707
Commit: 1c9b174663f4f73f7cce275e2d7b302c7d37d293
Parents: 869317b
Author: James Clampffer <ja...@hp.com>
Authored: Tue Jun 20 14:44:50 2017 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../src/main/native/libhdfspp/CMakeLists.txt | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c9b1746/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
index e0476cf..a5c77e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
@@ -16,6 +16,14 @@
# limitations under the License.
#
+# If cmake variable HDFSPP_LIBRARY_ONLY is set, then tests, examples, and
+# tools will not be built. This allows for faster builds of the libhdfspp
+# library alone, avoids looking for a JDK, valgrind, and gmock, and
+# prevents the generation of multiple binaries that might not be relevant
+# to other projects during normal use.
+# Example of cmake invocation with HDFSPP_LIBRARY_ONLY enabled:
+# cmake -DHDFSPP_LIBRARY_ONLY=1
+
project (libhdfspp)
cmake_minimum_required(VERSION 2.8)
@@ -174,9 +182,11 @@ include_directories( SYSTEM
add_subdirectory(third_party/gmock-1.7.0)
add_subdirectory(third_party/uriparser2)
add_subdirectory(lib)
-add_subdirectory(tests)
-add_subdirectory(examples)
-add_subdirectory(tools)
+if(NOT HDFSPP_LIBRARY_ONLY)
+ add_subdirectory(tests)
+ add_subdirectory(examples)
+ add_subdirectory(tools)
+endif()
# create an empty file; hadoop_add_dual_library wraps add_library which
# requires at least one file as an argument
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/52] [abbrv] hadoop git commit: HDFS-10746: libhdfs++: synchronize
access to working_directory and bytes_read_. Contributed by Anatoli Shein.
Posted by jh...@apache.org.
HDFS-10746: libhdfs++: synchronize access to working_directory and bytes_read_. Contributed by Anatoli Shein.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b9cf0e93
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b9cf0e93
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b9cf0e93
Branch: refs/heads/HDFS-8707
Commit: b9cf0e932d8a56eadc9c23912590722a5d846dec
Parents: cbb3e9f
Author: James <jh...@apache.org>
Authored: Mon Aug 15 12:21:25 2016 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Thu Mar 22 17:19:47 2018 -0400
----------------------------------------------------------------------
.../main/native/libhdfspp/lib/bindings/c/hdfs.cc | 17 ++++++++++++-----
.../src/main/native/libhdfspp/lib/fs/filehandle.cc | 4 ++--
.../src/main/native/libhdfspp/lib/fs/filehandle.h | 2 +-
3 files changed, 15 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9cf0e93/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
index 7fda4e2..4003358 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
@@ -43,18 +43,25 @@ static constexpr tPort kDefaultPort = 8020;
/* Separate the handles used by the C api from the C++ API*/
struct hdfs_internal {
- hdfs_internal(FileSystem *p) : filesystem_(p), working_directory("/") {}
+ hdfs_internal(FileSystem *p) : filesystem_(p), working_directory_("/") {}
hdfs_internal(std::unique_ptr<FileSystem> p)
- : filesystem_(std::move(p)), working_directory("/") {}
+ : filesystem_(std::move(p)), working_directory_("/") {}
virtual ~hdfs_internal(){};
FileSystem *get_impl() { return filesystem_.get(); }
const FileSystem *get_impl() const { return filesystem_.get(); }
- std::string get_working_directory() { return working_directory; }
- void set_working_directory(std::string new_directory) { working_directory = new_directory; }
+ std::string get_working_directory() {
+ std::lock_guard<std::mutex> read_guard(wd_lock_);
+ return working_directory_;
+ }
+ void set_working_directory(std::string new_directory) {
+ std::lock_guard<std::mutex> write_guard(wd_lock_);
+ working_directory_ = new_directory;
+ }
private:
std::unique_ptr<FileSystem> filesystem_;
- std::string working_directory; //has to always start and end with '/'
+ std::string working_directory_; //has to always start and end with '/'
+ std::mutex wd_lock_; //synchronize access to the working directory
};
struct hdfsFile_internal {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9cf0e93/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
index 2600944..8f1a82c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
@@ -353,8 +353,8 @@ bool FileHandle::ShouldExclude(const Status &s) {
}
}
-uint64_t FileHandleImpl::get_bytes_read() { return bytes_read_; }
+uint64_t FileHandleImpl::get_bytes_read() { return bytes_read_.load(); }
-void FileHandleImpl::clear_bytes_read() { bytes_read_ = 0; }
+void FileHandleImpl::clear_bytes_read() { bytes_read_.store(0); }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9cf0e93/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
index 03c55ff..7e7c79d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
@@ -139,7 +139,7 @@ private:
CancelHandle cancel_state_;
ReaderGroup readers_;
std::shared_ptr<LibhdfsEvents> event_handlers_;
- uint64_t bytes_read_;
+ std::atomic<uint64_t> bytes_read_;
};
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org