You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bo...@apache.org on 2016/02/04 14:10:47 UTC
[1/4] hadoop git commit: HDFS-9737: libhdfs++: Create examples of
consuming libhdfs++. Contributed by Bob Hansen
Repository: hadoop
Updated Branches:
refs/heads/HDFS-8707 2e0dd25cb -> 50817dc4c
HDFS-9737: libhdfs++: Create examples of consuming libhdfs++. Contributed by Bob Hansen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4f9ea2a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4f9ea2a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4f9ea2a3
Branch: refs/heads/HDFS-8707
Commit: 4f9ea2a34a992d01e7a34e877f23e2293eaa6d69
Parents: 2e0dd25
Author: Bob Hansen <bo...@hp.com>
Authored: Wed Feb 3 16:38:25 2016 -0500
Committer: Bob Hansen <bo...@hp.com>
Committed: Wed Feb 3 16:38:25 2016 -0500
----------------------------------------------------------------------
.../src/main/native/libhdfspp/CMakeLists.txt | 7 +++++++
.../src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h | 5 +++++
2 files changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f9ea2a3/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
index 8475000..834da71 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
@@ -124,6 +124,7 @@ include_directories( SYSTEM
add_subdirectory(third_party/gmock-1.7.0)
add_subdirectory(lib)
add_subdirectory(tests)
+add_subdirectory(examples)
# create an empty file; hadoop_add_dual_library wraps add_library which
# requires at least one file as an argument
@@ -170,3 +171,9 @@ install(FILES ${LIBHDFS_HEADER_FILES} DESTINATION /include/hdfs)
install(TARGETS hdfspp_static ARCHIVE DESTINATION /lib)
install(TARGETS hdfspp LIBRARY DESTINATION /lib)
+
+add_custom_target(
+ InstallToBuildDirectory
+ COMMAND "${CMAKE_MAKE_PROGRAM}" install DESTDIR=${PROJECT_BINARY_DIR}/output
+)
+set(LIBHDFSPP_DIR ${PROJECT_BINARY_DIR}/output)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f9ea2a3/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
index 3017fe1..1ddeba1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfs_ext.h
@@ -50,7 +50,9 @@
* Keep C bindings that are libhdfs++ specific in here.
**/
+#ifdef __cplusplus
extern "C" {
+#endif
/**
* Reads the last error, if any, that happened in this thread
* into the user supplied buffer.
@@ -108,5 +110,8 @@ int hdfsBuilderConfGetStr(struct hdfsBuilder *bld, const char *key,
*/
int hdfsBuilderConfGetInt(struct hdfsBuilder *bld, const char *key, int32_t *val);
+#ifdef __cplusplus
} /* end extern "C" */
#endif
+
+#endif
[3/4] hadoop git commit: HDFS-9741: libhdfs++: GetLastError not
returning meaningful messages after some failures. Contributed by Bob Hansen
Posted by bo...@apache.org.
HDFS-9741: libhdfs++: GetLastError not returning meaningful messages after some failures. Contributed by Bob Hansen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/038e01f5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/038e01f5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/038e01f5
Branch: refs/heads/HDFS-8707
Commit: 038e01f5e9e0e59e9234383d439e711dafa76817
Parents: 4d09d2d
Author: Bob Hansen <bo...@hp.com>
Authored: Wed Feb 3 16:41:15 2016 -0500
Committer: Bob Hansen <bo...@hp.com>
Committed: Wed Feb 3 16:41:15 2016 -0500
----------------------------------------------------------------------
.../native/libhdfspp/lib/bindings/c/hdfs.cc | 33 +++++++++++++++-----
1 file changed, 25 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/038e01f5/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
index 339c7fe..23e1f0a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
@@ -61,7 +61,7 @@ thread_local std::string errstr;
/* Fetch last error that happened in this thread */
void hdfsGetLastError(char *buf, int len) {
- if(nullptr == buf || len < 1 || errstr.empty()) {
+ if(nullptr == buf || len < 1) {
return;
}
@@ -93,7 +93,7 @@ struct hdfsBuilder {
};
/* Error handling with optional debug to stderr */
-static void ReportError(int errnum, std::string msg) {
+static void ReportError(int errnum, const std::string & msg) {
errno = errnum;
errstr = msg;
#ifdef LIBHDFSPP_C_API_ENABLE_DEBUG
@@ -106,24 +106,32 @@ static void ReportError(int errnum, std::string msg) {
/* Convert Status wrapped error into appropriate errno and return code */
static int Error(const Status &stat) {
+ const char * default_message;
+ int errnum;
+
int code = stat.code();
switch (code) {
case Status::Code::kOk:
return 0;
case Status::Code::kInvalidArgument:
- ReportError(EINVAL, "Invalid argument");
+ errnum = EINVAL;
+ default_message = "Invalid argument";
break;
case Status::Code::kResourceUnavailable:
- ReportError(EAGAIN, "Resource temporarily unavailable");
+ errnum = EAGAIN;
+ default_message = "Resource temporarily unavailable";
break;
case Status::Code::kUnimplemented:
- ReportError(ENOSYS, "Function not implemented");
+ errnum = ENOSYS;
+ default_message = "Function not implemented";
break;
case Status::Code::kException:
- ReportError(EINTR, "Exception raised");
+ errnum = EINTR;
+ default_message = "Exception raised";
break;
case Status::Code::kOperationCanceled:
- ReportError(EINTR, "Operation canceled");
+ errnum = EINTR;
+ default_message = "Operation canceled";
break;
case Status::Code::kPermissionDenied:
if (!stat.ToString().empty())
@@ -132,8 +140,13 @@ static int Error(const Status &stat) {
ReportError(EACCES, "Permission denied");
break;
default:
- ReportError(ENOSYS, "Error: unrecognised code");
+ errnum = ENOSYS;
+ default_message = "Error: unrecognised code";
}
+ if (stat.ToString().empty())
+ ReportError(errnum, default_message);
+ else
+ ReportError(errnum, stat.ToString());
return -1;
}
@@ -176,6 +189,7 @@ hdfsFS hdfsConnectAsUser(const char* nn, tPort port, const char *user) {
FileSystem *fs = FileSystem::New(io_service, user_name, Options());
if (!fs) {
+ ReportError(ENODEV, "Could not create FileSystem object");
return nullptr;
}
@@ -217,6 +231,7 @@ hdfsFile hdfsOpenFile(hdfsFS fs, const char *path, int flags, int bufferSize,
FileHandle *f = nullptr;
Status stat = fs->get_impl()->Open(path, &f);
if (!stat.ok()) {
+ Error(stat);
return nullptr;
}
return new hdfsFile_internal(f);
@@ -365,6 +380,7 @@ int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key,
}
else
{
+ ReportError(EINVAL, "Could not change Builder value");
return 1;
}
}
@@ -454,5 +470,6 @@ int hdfsBuilderConfGetInt(struct hdfsBuilder *bld, const char *key, int32_t *val
*val = *value;
}
// If not found, don't change val
+ ReportError(EINVAL, "Could not get Builder value");
return 0;
}
[4/4] hadoop git commit: HDFS-9749: libhdfs++: RPC engine will
attempt to close an asio socket before it's been opened. Contributed by James
Clampffer
Posted by bo...@apache.org.
HDFS-9749: libhdfs++: RPC engine will attempt to close an asio socket before it's been opened. Contributed by James Clampffer
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/50817dc4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/50817dc4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/50817dc4
Branch: refs/heads/HDFS-8707
Commit: 50817dc4ceecfe49789dc9718ef1dcaf80197b52
Parents: 038e01f
Author: Bob Hansen <bo...@hp.com>
Authored: Wed Feb 3 16:41:59 2016 -0500
Committer: Bob Hansen <bo...@hp.com>
Committed: Wed Feb 3 16:41:59 2016 -0500
----------------------------------------------------------------------
.../src/main/native/libhdfspp/lib/rpc/rpc_connection.h | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50817dc4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
index 61d62e9..a8820c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
@@ -282,8 +282,10 @@ void RpcConnectionImpl<NextLayer>::Disconnect() {
assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
request_over_the_wire_.reset();
- next_layer_.cancel();
- next_layer_.close();
+ if (connected_) {
+ next_layer_.cancel();
+ next_layer_.close();
+ }
connected_ = false;
}
}
[2/4] hadoop git commit: HDFS-9738: libhdfs++: Implement simple
authentication. Contributed by Bob Hansen
Posted by bo...@apache.org.
HDFS-9738: libhdfs++: Implement simple authentication. Contributed by Bob Hansen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4d09d2dc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4d09d2dc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4d09d2dc
Branch: refs/heads/HDFS-8707
Commit: 4d09d2dcf3e51feb466b1ef80b2ca0fa887b9273
Parents: 4f9ea2a
Author: Bob Hansen <bo...@hp.com>
Authored: Wed Feb 3 16:40:07 2016 -0500
Committer: Bob Hansen <bo...@hp.com>
Committed: Wed Feb 3 16:40:07 2016 -0500
----------------------------------------------------------------------
.../native/libhdfspp/include/hdfspp/hdfspp.h | 4 +-
.../native/libhdfspp/include/hdfspp/status.h | 1 +
.../native/libhdfspp/lib/bindings/c/hdfs.cc | 33 ++++++++++++++++-
.../main/native/libhdfspp/lib/common/status.cc | 8 +++-
.../main/native/libhdfspp/lib/fs/filesystem.cc | 39 ++++++++++++++++++--
.../main/native/libhdfspp/lib/fs/filesystem.h | 8 ++--
.../native/libhdfspp/lib/rpc/rpc_connection.cc | 19 ++++++++++
.../main/native/libhdfspp/lib/rpc/rpc_engine.cc | 5 ++-
.../main/native/libhdfspp/lib/rpc/rpc_engine.h | 7 +++-
.../native/libhdfspp/tests/rpc_engine_test.cc | 18 ++++-----
10 files changed, 117 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d09d2dc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
index 2cbb62c..2451b16 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
@@ -120,9 +120,11 @@ class FileSystem {
* Create a new instance of the FileSystem object. The call
* initializes the RPC connections to the NameNode and returns an
* FileSystem object.
+ *
+ * If user_name is blank, the current user will be used for a default.
**/
static FileSystem * New(
- IoService *&io_service, const Options &options);
+ IoService *&io_service, const std::string &user_name, const Options &options);
virtual void Connect(const std::string &server,
const std::string &service,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d09d2dc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
index a91ac9d..3c7563d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
@@ -54,6 +54,7 @@ class Status {
kResourceUnavailable = static_cast<unsigned>(std::errc::resource_unavailable_try_again),
kUnimplemented = static_cast<unsigned>(std::errc::function_not_supported),
kOperationCanceled = static_cast<unsigned>(std::errc::operation_canceled),
+ kPermissionDenied = static_cast<unsigned>(std::errc::permission_denied),
kException = 255,
};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d09d2dc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
index 3262c66..339c7fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
@@ -86,6 +86,7 @@ struct hdfsBuilder {
std::string overrideHost;
tPort overridePort; // 0 --> use default
+ std::string user;
static constexpr tPort kUseDefaultPort = 0;
static constexpr tPort kDefaultPort = 8020;
@@ -124,6 +125,12 @@ static int Error(const Status &stat) {
case Status::Code::kOperationCanceled:
ReportError(EINTR, "Operation canceled");
break;
+ case Status::Code::kPermissionDenied:
+ if (!stat.ToString().empty())
+ ReportError(EACCES, stat.ToString().c_str());
+ else
+ ReportError(EACCES, "Permission denied");
+ break;
default:
ReportError(ENOSYS, "Error: unrecognised code");
}
@@ -156,9 +163,18 @@ int hdfsFileIsOpenForRead(hdfsFile file) {
}
hdfsFS hdfsConnect(const char *nn, tPort port) {
+ return hdfsConnectAsUser(nn, port, "");
+}
+
+hdfsFS hdfsConnectAsUser(const char* nn, tPort port, const char *user) {
std::string port_as_string = std::to_string(port);
IoService * io_service = IoService::New();
- FileSystem *fs = FileSystem::New(io_service, Options());
+ std::string user_name;
+ if (user) {
+ user_name = user;
+ }
+
+ FileSystem *fs = FileSystem::New(io_service, user_name, Options());
if (!fs) {
return nullptr;
}
@@ -323,6 +339,16 @@ void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port)
bld->overridePort = port;
}
+void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName)
+{
+ if (userName) {
+ bld->user = userName;
+ } else {
+ bld->user = "";
+ }
+}
+
+
void hdfsFreeBuilder(struct hdfsBuilder *bld)
{
delete bld;
@@ -358,7 +384,10 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) {
{
port = hdfsBuilder::kDefaultPort;
}
- return hdfsConnect(bld->overrideHost.c_str(), port);
+ if (bld->user.empty())
+ return hdfsConnect(bld->overrideHost.c_str(), port);
+ else
+ return hdfsConnectAsUser(bld->overrideHost.c_str(), port, bld->user.c_str());
}
else
{
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d09d2dc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
index eb22247..78d4379 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/status.cc
@@ -20,9 +20,12 @@
#include <cassert>
#include <sstream>
+#include <cstring>
namespace hdfs {
+const char * kStatusAccessControlException = "org.apache.hadoop.security.AccessControlException";
+
Status::Status(int code, const char *msg1) : code_(code) {
if(msg1) {
msg_ = msg1;
@@ -58,7 +61,10 @@ Status Status::Unimplemented() {
}
Status Status::Exception(const char *exception_class_name, const char *error_message) {
- return Status(kException, exception_class_name, error_message);
+ if (exception_class_name && (strcmp(exception_class_name, kStatusAccessControlException) == 0) )
+ return Status(kPermissionDenied, error_message);
+ else
+ return Status(kException, exception_class_name, error_message);
}
Status Status::Error(const char *error_message) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d09d2dc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
index fafaa1b..0ced667 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
@@ -27,6 +27,7 @@
#include <future>
#include <tuple>
#include <iostream>
+#include <pwd.h>
namespace hdfs {
@@ -108,18 +109,48 @@ void NameNodeOperations::GetBlockLocations(const std::string & path,
****************************************************************************/
FileSystem * FileSystem::New(
- IoService *&io_service, const Options &options) {
- return new FileSystemImpl(io_service, options);
+ IoService *&io_service, const std::string &user_name, const Options &options) {
+ return new FileSystemImpl(io_service, user_name, options);
}
/*****************************************************************************
* FILESYSTEM IMPLEMENTATION
****************************************************************************/
-FileSystemImpl::FileSystemImpl(IoService *&io_service, const Options &options)
+const std::string get_effective_user_name(const std::string &user_name) {
+ if (!user_name.empty())
+ return user_name;
+
+ // If no user name was provided, try the HADOOP_USER_NAME and USER environment
+ // variables
+ const char * env = getenv("HADOOP_USER_NAME");
+ if (env) {
+ return env;
+ }
+
+ env = getenv("USER");
+ if (env) {
+ return env;
+ }
+
+ // If running on POSIX, use the currently logged in user
+#if defined(_POSIX_VERSION)
+ uid_t uid = geteuid();
+ struct passwd *pw = getpwuid(uid);
+ if (pw && pw->pw_name)
+ {
+ return pw->pw_name;
+ }
+#endif
+
+ return "unknown_user";
+}
+
+FileSystemImpl::FileSystemImpl(IoService *&io_service, const std::string &user_name,
+ const Options &options)
: io_service_(static_cast<IoServiceImpl *>(io_service)),
nn_(&io_service_->io_service(), options,
- GetRandomClientName(), kNamenodeProtocol,
+ GetRandomClientName(), get_effective_user_name(user_name), kNamenodeProtocol,
kNamenodeProtocolVersion), client_name_(GetRandomClientName()),
bad_node_tracker_(std::make_shared<BadDataNodeTracker>())
{
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d09d2dc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
index 0ae032d..726b8d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
@@ -47,10 +47,10 @@ namespace hdfs {
class NameNodeOperations {
public:
NameNodeOperations(::asio::io_service *io_service, const Options &options,
- const std::string &client_name, const char *protocol_name,
- int protocol_version) :
+ const std::string &client_name, const std::string &user_name,
+ const char *protocol_name, int protocol_version) :
io_service_(io_service),
- engine_(io_service, options, client_name, protocol_name, protocol_version),
+ engine_(io_service, options, client_name, user_name, protocol_name, protocol_version),
namenode_(& engine_) {}
void Connect(const std::string &server,
@@ -80,7 +80,7 @@ private:
*/
class FileSystemImpl : public FileSystem {
public:
- FileSystemImpl(IoService *&io_service, const Options &options);
+ FileSystemImpl(IoService *&io_service, const std::string& user_name, const Options &options);
~FileSystemImpl() override;
/* attempt to connect to namenode, return bad status on failure */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d09d2dc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
index 51f0420..91d8667 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.cc
@@ -227,6 +227,21 @@ void RpcConnection::HandleRpcTimeout(std::shared_ptr<Request> req,
std::shared_ptr<std::string> RpcConnection::PrepareHandshakePacket() {
assert(lock_held(connection_state_lock_)); // Must be holding lock before calling
+ /** From Client.java:
+ *
+ * Write the connection header - this is sent when connection is established
+ * +----------------------------------+
+ * | "hrpc" 4 bytes |
+ * +----------------------------------+
+ * | Version (1 byte) |
+ * +----------------------------------+
+ * | Service Class (1 byte) |
+ * +----------------------------------+
+ * | AuthProtocol (1 byte) |
+ * +----------------------------------+
+ *
+ * AuthProtocol: 0->none, -33->SASL
+ */
static const char kHandshakeHeader[] = {'h', 'r', 'p', 'c',
RpcEngine::kRpcVersion, 0, 0};
auto res =
@@ -240,6 +255,10 @@ std::shared_ptr<std::string> RpcConnection::PrepareHandshakePacket() {
IpcConnectionContextProto handshake;
handshake.set_protocol(engine_->protocol_name());
+ const std::string & user_name = engine()->user_name();
+ if (!user_name.empty()) {
+ *handshake.mutable_userinfo()->mutable_effectiveuser() = user_name;
+ }
AddHeadersToPacket(res.get(), {&h, &handshake}, nullptr);
return res;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d09d2dc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
index a84257b..8d3e404 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
@@ -28,11 +28,12 @@ template <class T>
using optional = std::experimental::optional<T>;
RpcEngine::RpcEngine(::asio::io_service *io_service, const Options &options,
- const std::string &client_name, const char *protocol_name,
- int protocol_version)
+ const std::string &client_name, const std::string &user_name,
+ const char *protocol_name, int protocol_version)
: io_service_(io_service),
options_(options),
client_name_(client_name),
+ user_name_(user_name),
protocol_name_(protocol_name),
protocol_version_(protocol_version),
retry_policy_(std::move(MakeRetryPolicy(options))),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d09d2dc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
index fb1844d..75d4e67 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
@@ -206,6 +206,7 @@ public:
virtual int NextCallId() = 0;
virtual const std::string &client_name() const = 0;
+ virtual const std::string &user_name() const = 0;
virtual const std::string &protocol_name() const = 0;
virtual int protocol_version() const = 0;
virtual ::asio::io_service &io_service() = 0;
@@ -230,8 +231,8 @@ class RpcEngine : public LockFreeRpcEngine {
};
RpcEngine(::asio::io_service *io_service, const Options &options,
- const std::string &client_name, const char *protocol_name,
- int protocol_version);
+ const std::string &client_name, const std::string &user_name,
+ const char *protocol_name, int protocol_version);
void Connect(const std::vector<::asio::ip::tcp::endpoint> &server, RpcCallback &handler);
@@ -265,6 +266,7 @@ class RpcEngine : public LockFreeRpcEngine {
void TEST_SetRpcConnection(std::shared_ptr<RpcConnection> conn);
const std::string &client_name() const override { return client_name_; }
+ const std::string &user_name() const override { return user_name_; }
const std::string &protocol_name() const override { return protocol_name_; }
int protocol_version() const override { return protocol_version_; }
::asio::io_service &io_service() override { return *io_service_; }
@@ -281,6 +283,7 @@ private:
::asio::io_service * const io_service_;
const Options options_;
const std::string client_name_;
+ const std::string user_name_;
const std::string protocol_name_;
const int protocol_version_;
const std::unique_ptr<const RetryPolicy> retry_policy_; //null --> no retry
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d09d2dc/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
index 28c7596..de9972e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
@@ -105,7 +105,7 @@ using namespace hdfs;
TEST(RpcEngineTest, TestRoundTrip) {
::asio::io_service io_service;
Options options;
- RpcEngine engine(&io_service, options, "foo", "protocol", 1);
+ RpcEngine engine(&io_service, options, "foo", "", "protocol", 1);
RpcConnectionImpl<MockRPCConnection> *conn =
new RpcConnectionImpl<MockRPCConnection>(&engine);
conn->TEST_set_connected(true);
@@ -141,7 +141,7 @@ TEST(RpcEngineTest, TestRoundTrip) {
TEST(RpcEngineTest, TestConnectionResetAndFail) {
::asio::io_service io_service;
Options options;
- RpcEngine engine(&io_service, options, "foo", "protocol", 1);
+ RpcEngine engine(&io_service, options, "foo", "", "protocol", 1);
RpcConnectionImpl<MockRPCConnection> *conn =
new RpcConnectionImpl<MockRPCConnection>(&engine);
conn->TEST_set_connected(true);
@@ -178,7 +178,7 @@ TEST(RpcEngineTest, TestConnectionResetAndRecover) {
Options options;
options.max_rpc_retries = 1;
options.rpc_retry_delay_ms = 0;
- SharedConnectionEngine engine(&io_service, options, "foo", "protocol", 1);
+ SharedConnectionEngine engine(&io_service, options, "foo", "", "protocol", 1);
EchoResponseProto server_resp;
server_resp.set_message("foo");
@@ -213,7 +213,7 @@ TEST(RpcEngineTest, TestConnectionResetAndRecoverWithDelay) {
Options options;
options.max_rpc_retries = 1;
options.rpc_retry_delay_ms = 1;
- SharedConnectionEngine engine(&io_service, options, "foo", "protocol", 1);
+ SharedConnectionEngine engine(&io_service, options, "foo", "", "protocol", 1);
EchoResponseProto server_resp;
server_resp.set_message("foo");
@@ -262,7 +262,7 @@ TEST(RpcEngineTest, TestConnectionFailure)
Options options;
options.max_rpc_retries = 0;
options.rpc_retry_delay_ms = 0;
- SharedConnectionEngine engine(&io_service, options, "foo", "protocol", 1);
+ SharedConnectionEngine engine(&io_service, options, "foo", "", "protocol", 1);
EXPECT_CALL(*producer, Produce())
.WillOnce(Return(std::make_pair(make_error_code(::asio::error::connection_reset), "")));
@@ -288,7 +288,7 @@ TEST(RpcEngineTest, TestConnectionFailureRetryAndFailure)
Options options;
options.max_rpc_retries = 2;
options.rpc_retry_delay_ms = 0;
- SharedConnectionEngine engine(&io_service, options, "foo", "protocol", 1);
+ SharedConnectionEngine engine(&io_service, options, "foo", "", "protocol", 1);
EXPECT_CALL(*producer, Produce())
.WillOnce(Return(std::make_pair(make_error_code(::asio::error::connection_reset), "")))
.WillOnce(Return(std::make_pair(make_error_code(::asio::error::connection_reset), "")))
@@ -316,7 +316,7 @@ TEST(RpcEngineTest, TestConnectionFailureAndRecover)
Options options;
options.max_rpc_retries = 1;
options.rpc_retry_delay_ms = 0;
- SharedConnectionEngine engine(&io_service, options, "foo", "protocol", 1);
+ SharedConnectionEngine engine(&io_service, options, "foo", "", "protocol", 1);
EXPECT_CALL(*producer, Produce())
.WillOnce(Return(std::make_pair(make_error_code(::asio::error::connection_reset), "")))
.WillOnce(Return(std::make_pair(::asio::error_code(), "")))
@@ -345,7 +345,7 @@ TEST(RpcEngineTest, TestConnectionFailureAndAsyncRecover)
Options options;
options.max_rpc_retries = 1;
options.rpc_retry_delay_ms = 1;
- SharedConnectionEngine engine(&io_service, options, "foo", "protocol", 1);
+ SharedConnectionEngine engine(&io_service, options, "foo", "", "protocol", 1);
EXPECT_CALL(*producer, Produce())
.WillOnce(Return(std::make_pair(make_error_code(::asio::error::connection_reset), "")))
.WillOnce(Return(std::make_pair(::asio::error_code(), "")))
@@ -369,7 +369,7 @@ TEST(RpcEngineTest, TestTimeout) {
::asio::io_service io_service;
Options options;
options.rpc_timeout = 1;
- RpcEngine engine(&io_service, options, "foo", "protocol", 1);
+ RpcEngine engine(&io_service, options, "foo", "", "protocol", 1);
RpcConnectionImpl<MockRPCConnection> *conn =
new RpcConnectionImpl<MockRPCConnection>(&engine);
conn->TEST_set_connected(true);