You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bo...@apache.org on 2016/03/14 16:35:58 UTC
[5/5] hadoop git commit: HDFS-9556: libhdfs++: pull Options from
default configs by default. Contributed by Bob Hansen.
HDFS-9556: libhdfs++: pull Options from default configs by default. Contributed by Bob Hansen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/77515072
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/77515072
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/77515072
Branch: refs/heads/HDFS-8707
Commit: 7751507233dc7888c84c5fb8f864299782a6226f
Parents: 196db0d
Author: Bob Hansen <bo...@hp.com>
Authored: Mon Mar 14 11:34:42 2016 -0400
Committer: Bob Hansen <bo...@hp.com>
Committed: Mon Mar 14 11:34:42 2016 -0400
----------------------------------------------------------------------
.../src/main/native/libhdfspp/CMakeLists.txt | 3 +-
.../native/libhdfspp/include/hdfspp/hdfspp.h | 13 +-
.../native/libhdfspp/include/hdfspp/options.h | 7 +
.../native/libhdfspp/lib/bindings/c/hdfs.cc | 89 ++---
.../native/libhdfspp/lib/common/CMakeLists.txt | 4 +-
.../libhdfspp/lib/common/configuration.cc | 27 ++
.../native/libhdfspp/lib/common/configuration.h | 5 +
.../lib/common/configuration_loader.cc | 12 +-
.../libhdfspp/lib/common/configuration_loader.h | 2 +-
.../libhdfspp/lib/common/hdfs_configuration.cc | 1 +
.../main/native/libhdfspp/lib/common/options.cc | 4 +-
.../src/main/native/libhdfspp/lib/common/uri.cc | 371 +++++++++++++++++++
.../src/main/native/libhdfspp/lib/common/uri.h | 128 +++++++
.../main/native/libhdfspp/lib/fs/filesystem.cc | 50 ++-
.../main/native/libhdfspp/lib/fs/filesystem.h | 7 +-
.../main/native/libhdfspp/tests/CMakeLists.txt | 4 +
.../libhdfspp/tests/configuration_test.cc | 37 ++
.../src/main/native/libhdfspp/tests/uri_test.cc | 245 ++++++++++++
.../third_party/uriparser2/CMakeLists.txt | 2 +-
19 files changed, 946 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77515072/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
index 2766cf7..6f248f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
@@ -46,6 +46,7 @@ set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0")
if(UNIX)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic -std=c++11 -g -fPIC -fno-strict-aliasing")
+set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -fPIC -fno-strict-aliasing")
endif()
# Mac OS 10.7 and later deprecates most of the methods in OpenSSL.
@@ -138,7 +139,7 @@ if(NEED_LINK_DL)
set(LIB_DL dl)
endif()
-set(LIBHDFSPP_ALL_OBJECTS $<TARGET_OBJECTS:bindings_c_obj> $<TARGET_OBJECTS:fs_obj> $<TARGET_OBJECTS:rpc_obj> $<TARGET_OBJECTS:reader_obj> $<TARGET_OBJECTS:proto_obj> $<TARGET_OBJECTS:connection_obj> $<TARGET_OBJECTS:common_obj>)
+set(LIBHDFSPP_ALL_OBJECTS $<TARGET_OBJECTS:bindings_c_obj> $<TARGET_OBJECTS:fs_obj> $<TARGET_OBJECTS:rpc_obj> $<TARGET_OBJECTS:reader_obj> $<TARGET_OBJECTS:proto_obj> $<TARGET_OBJECTS:connection_obj> $<TARGET_OBJECTS:common_obj> $<TARGET_OBJECTS:uriparser2_obj>)
if (HADOOP_BUILD)
hadoop_add_dual_library(hdfspp ${EMPTY_FILE_CC} ${LIBHDFSPP_ALL_OBJECTS})
hadoop_target_link_dual_libraries(hdfspp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77515072/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
index 2451b16..b52e832 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
@@ -128,12 +128,23 @@ class FileSystem {
virtual void Connect(const std::string &server,
const std::string &service,
- const std::function<void(const Status &, FileSystem *)> &&handler) = 0;
+ const std::function<void(const Status &, FileSystem *)> &handler) = 0;
/* Synchronous call of Connect */
virtual Status Connect(const std::string &server,
const std::string &service) = 0;
+
+ /**
+ * Connects to the hdfs instance indicated by the defaultFs value of the
+ * Options structure.
+ *
+ * If no defaultFs is defined, returns an error.
+ */
+ virtual void ConnectToDefaultFs(
+ const std::function<void(const Status &, FileSystem *)> &handler) = 0;
+ virtual Status ConnectToDefaultFs() = 0;
+
/**
* Open a file on HDFS. The call issues an RPC to the NameNode to
* gather the locations of all blocks in the file and to return a
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77515072/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
index 6a3799a..f63e70b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
@@ -18,6 +18,8 @@
#ifndef LIBHDFSPP_OPTIONS_H_
#define LIBHDFSPP_OPTIONS_H_
+#include "common/uri.h"
+
namespace hdfs {
/**
@@ -51,6 +53,11 @@ struct Options {
unsigned int host_exclusion_duration;
static const unsigned int kDefaultHostExclusionDuration = 600000;
+ /**
+ * URI to connect to if no host:port are specified in connect
+ */
+ URI defaultFS;
+
Options();
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77515072/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
index 03f3140..fd46f3f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
@@ -31,6 +31,9 @@
#include <algorithm>
using namespace hdfs;
+using std::experimental::nullopt;
+
+static constexpr tPort kDefaultPort = 8020;
/* Separate the handles used by the C api from the C++ API*/
struct hdfs_internal {
@@ -84,12 +87,11 @@ struct hdfsBuilder {
ConfigurationLoader loader;
HdfsConfiguration config;
- std::string overrideHost;
- tPort overridePort; // 0 --> use default
- std::string user;
+ optional<std::string> overrideHost;
+ optional<tPort> overridePort;
+ optional<std::string> user;
static constexpr tPort kUseDefaultPort = 0;
- static constexpr tPort kDefaultPort = 8020;
};
/* Error handling with optional debug to stderr */
@@ -183,28 +185,30 @@ int hdfsFileIsOpenForRead(hdfsFile file) {
return 0;
}
-hdfsFS hdfsConnect(const char *nn, tPort port) {
- return hdfsConnectAsUser(nn, port, "");
-}
-
-hdfsFS hdfsConnectAsUser(const char* nn, tPort port, const char *user) {
+hdfsFS doHdfsConnect(optional<std::string> nn, optional<tPort> port, optional<std::string> user, const Options & options) {
try
{
- std::string port_as_string = std::to_string(port);
IoService * io_service = IoService::New();
- std::string user_name;
- if (user) {
- user_name = user;
- }
- FileSystem *fs = FileSystem::New(io_service, user_name, Options());
+ FileSystem *fs = FileSystem::New(io_service, user.value_or(""), options);
if (!fs) {
ReportError(ENODEV, "Could not create FileSystem object");
return nullptr;
}
- if (!fs->Connect(nn, port_as_string).ok()) {
- ReportError(ENODEV, "Unable to connect to NameNode.");
+ Status status;
+ if (nn || port) {
+ if (!port) {
+ port = kDefaultPort;
+ }
+ std::string port_as_string = std::to_string(*port);
+ status = fs->Connect(nn.value_or(""), port_as_string);
+ } else {
+ status = fs->ConnectToDefaultFs();
+ }
+
+ if (!status.ok()) {
+ Error(status);
// FileSystem's ctor might take ownership of the io_service; if it does,
// it will null out the pointer
@@ -225,6 +229,14 @@ hdfsFS hdfsConnectAsUser(const char* nn, tPort port, const char *user) {
}
}
+hdfsFS hdfsConnect(const char *nn, tPort port) {
+ return hdfsConnectAsUser(nn, port, "");
+}
+
+hdfsFS hdfsConnectAsUser(const char* nn, tPort port, const char *user) {
+ return doHdfsConnect(std::string(nn), port, std::string(user), Options());
+}
+
int hdfsDisconnect(hdfsFS fs) {
try
{
@@ -403,12 +415,14 @@ HdfsConfiguration LoadDefault(ConfigurationLoader & loader)
}
}
-hdfsBuilder::hdfsBuilder() : config(LoadDefault(loader)), overridePort(kUseDefaultPort)
+hdfsBuilder::hdfsBuilder() : config(loader.New<HdfsConfiguration>())
{
+ loader.SetDefaultSearchPath();
+ config = LoadDefault(loader);
}
hdfsBuilder::hdfsBuilder(const char * directory) :
- config(loader.New<HdfsConfiguration>()), overridePort(kUseDefaultPort)
+ config(loader.New<HdfsConfiguration>())
{
loader.SetSearchPath(directory);
config = LoadDefault(loader);
@@ -430,7 +444,7 @@ struct hdfsBuilder *hdfsNewBuilder(void)
void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn)
{
- bld->overrideHost = nn;
+ bld->overrideHost = std::string(nn);
}
void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port)
@@ -440,10 +454,8 @@ void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port)
void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName)
{
- if (userName) {
- bld->user = userName;
- } else {
- bld->user = "";
+ if (userName && *userName) {
+ bld->user = std::string(userName);
}
}
@@ -489,34 +501,7 @@ void hdfsConfStrFree(char *val)
}
hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) {
- try
- {
- if (!bld->overrideHost.empty())
- {
- // TODO: pass rest of config once we get that done (HDFS-9556)
- tPort port = bld->overridePort;
- if (port == hdfsBuilder::kUseDefaultPort)
- {
- port = hdfsBuilder::kDefaultPort;
- }
- if (bld->user.empty())
- return hdfsConnect(bld->overrideHost.c_str(), port);
- else
- return hdfsConnectAsUser(bld->overrideHost.c_str(), port, bld->user.c_str());
- }
- else
- {
- //TODO: allow construction from default port once that is done (HDFS-9556)
- ReportError(EINVAL, "No host provided to builder in hdfsBuilderConnect");
- return nullptr;
- }
- } catch (const std::exception & e) {
- ReportException(e);
- return nullptr;
- } catch (...) {
- ReportCaughtNonException();
- return nullptr;
- }
+ return doHdfsConnect(bld->overrideHost, bld->overridePort, bld->user, bld->config.GetOptions());
}
int hdfsConfGetStr(const char *key, char **val)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77515072/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
index 0344a7c..2e78643 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
@@ -19,6 +19,6 @@ if(NEED_LINK_DL)
set(LIB_DL dl)
endif()
-add_library(common_obj OBJECT base64.cc status.cc sasl_digest_md5.cc hdfs_public_api.cc options.cc configuration.cc configuration_loader.cc hdfs_configuration.cc util.cc retry_policy.cc cancel_tracker.cc)
-add_library(common $<TARGET_OBJECTS:common_obj>)
+add_library(common_obj OBJECT base64.cc status.cc sasl_digest_md5.cc hdfs_public_api.cc options.cc configuration.cc configuration_loader.cc hdfs_configuration.cc uri.cc util.cc retry_policy.cc cancel_tracker.cc)
+add_library(common $<TARGET_OBJECTS:common_obj> $<TARGET_OBJECTS:uriparser2_obj>)
target_link_libraries(common ${LIB_DL})
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77515072/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.cc
index be73be3..ac9a089 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.cc
@@ -32,6 +32,7 @@
*/
#include "configuration.h"
+#include "uri.h"
#include <strings.h>
#include <sstream>
@@ -137,4 +138,30 @@ bool Configuration::GetBoolWithDefault(const std::string& key,
bool default_value) const {
return GetBool(key).value_or(default_value);
}
+
+optional<URI> Configuration::GetUri(const std::string& key) const {
+ auto raw = Get(key);
+ if (raw) {
+ return URI::parse_from_string(*raw);
+ } else {
+ return optional<URI>();
+ }
+}
+
+URI Configuration::GetUriWithDefault(const std::string& key,
+ std::string default_value) const {
+ optional<URI> result = GetUri(key);
+ if (result) {
+ return *result;
+ } else {
+ result = URI::parse_from_string(default_value);
+ if (result) {
+ return *result;
+ } else {
+ return URI();
+ }
+ }
+}
+
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77515072/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.h
index 81a8934..b6b968e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.h
@@ -19,6 +19,8 @@
#ifndef COMMON_CONFIGURATION_H_
#define COMMON_CONFIGURATION_H_
+#include "common/uri.h"
+
#include <string>
#include <map>
#include <vector>
@@ -63,6 +65,9 @@ class Configuration {
bool GetBoolWithDefault(const std::string &key,
bool default_value) const;
optional<bool> GetBool(const std::string &key) const;
+ URI GetUriWithDefault(const std::string &key,
+ std::string default_value) const;
+ optional<URI> GetUri(const std::string &key) const;
protected:
friend class ConfigurationLoader;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77515072/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.cc
index 9bcf7e4..1eb70c3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.cc
@@ -63,9 +63,15 @@ bool str_to_bool(const std::string& raw) {
}
void ConfigurationLoader::SetDefaultSearchPath() {
- //TODO: Use HADOOP_CONF_DIR when we get environment subs with HDFS-9385
- AddToSearchPath("./");
- AddToSearchPath("/etc/hadoop/");
+ // Try (in order, taking the first valid one):
+ // $HADOOP_CONF_DIR
+ // /etc/hadoop/conf
+ const char * hadoop_conf_dir_env = getenv("HADOOP_CONF_DIR");
+ if (hadoop_conf_dir_env) {
+ AddToSearchPath(hadoop_conf_dir_env);
+ } else {
+ AddToSearchPath("/etc/hadoop/conf");
+ }
}
void ConfigurationLoader::ClearSearchPath()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77515072/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.h
index 852031d..059e48b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.h
@@ -79,7 +79,7 @@ public:
* SEARCH PATH METHODS
***************************************************************************/
- // Sets the search path to the default search path (namely, ".:/etc/hadoop")
+ // Sets the search path to the default search path (namely, "$HADOOP_CONF_DIR" or "/etc/hadoop/conf")
void SetDefaultSearchPath();
// Clears out the search path
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77515072/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc
index f34ac44..6d50a29 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_configuration.cc
@@ -47,6 +47,7 @@ Options HdfsConfiguration::GetOptions() {
OptionalSet(result.rpc_timeout, GetInt(kDfsClientSocketTimeoutKey));
OptionalSet(result.max_rpc_retries, GetInt(kIpcClientConnectMaxRetriesKey));
OptionalSet(result.rpc_retry_delay_ms, GetInt(kIpcClientConnectRetryIntervalKey));
+ OptionalSet(result.defaultFS, GetUri(kFsDefaultFsKey));
return result;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77515072/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc
index c7d418f..6406dea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/options.cc
@@ -29,5 +29,7 @@ const unsigned int Options::kDefaultHostExclusionDuration;
Options::Options() : rpc_timeout(kDefaultRpcTimeout), max_rpc_retries(kDefaultMaxRpcRetries),
rpc_retry_delay_ms(kDefaultRpcRetryDelayMs),
- host_exclusion_duration(kDefaultHostExclusionDuration) {}
+ host_exclusion_duration(kDefaultHostExclusionDuration),
+ defaultFS()
+{}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77515072/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.cc
new file mode 100644
index 0000000..616cf3f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.cc
@@ -0,0 +1,371 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#include <common/uri.h>
+
+#include <uriparser2/uriparser/Uri.h>
+
+#include <string.h>
+#include <sstream>
+#include <cstdlib>
+#include <limits>
+
+using std::experimental::nullopt;
+
+namespace hdfs
+{
+
+///////////////////////////////////////////////////////////////////////////////
+//
+// Internal utilities
+//
+///////////////////////////////////////////////////////////////////////////////
+
+const char kReserved[] = ":/?#[]@%+";
+
+std::string URI::encode(const std::string & decoded)
+{
+ bool hasCharactersToEncode = false;
+ for (auto c : decoded)
+ {
+ if (isalnum(c) || (strchr(kReserved, c) == NULL))
+ {
+ continue;
+ }
+ else
+ {
+ hasCharactersToEncode = true;
+ break;
+ }
+ }
+
+ if (hasCharactersToEncode)
+ {
+ std::vector<char> buf(decoded.size() * 3 + 1);
+ uriEscapeA(decoded.c_str(), &buf[0], true, URI_BR_DONT_TOUCH);
+ return std::string(&buf[0]);
+ }
+ else
+ {
+ return decoded;
+ }
+}
+
+std::string URI::decode(const std::string & encoded)
+{
+ bool hasCharactersToDecode = false;
+ for (auto c : encoded)
+ {
+ switch (c)
+ {
+ case '%':
+ case '+':
+ hasCharactersToDecode = true;
+ break;
+ default:
+ continue;
+ }
+ }
+
+ if (hasCharactersToDecode)
+ {
+ std::vector<char> buf(encoded.size() + 1);
+ strncpy(&buf[0], encoded.c_str(), buf.size());
+ uriUnescapeInPlaceExA(&buf[0], true, URI_BR_DONT_TOUCH);
+ return std::string(&buf[0]);
+ }
+ else
+ {
+ return encoded;
+ }
+}
+
+std::vector<std::string> split(const std::string input, char separator)
+{
+ std::vector<std::string> result;
+
+ if (!input.empty())
+ {
+ const char * remaining = input.c_str();
+ if (*remaining == '/')
+ remaining++;
+
+ const char * next_end = strchr(remaining, separator);
+ while (next_end) {
+ int len = next_end - remaining;
+ if (len)
+ result.push_back(std::string(remaining, len));
+ else
+ result.push_back("");
+ remaining = next_end + 1;
+ next_end = strchr(remaining, separator);
+ }
+ result.push_back(std::string(remaining));
+ }
+
+ return result;
+}
+
+
+
+///////////////////////////////////////////////////////////////////////////////
+//
+// Parsing
+//
+///////////////////////////////////////////////////////////////////////////////
+
+
+
+std::string copy_range(const UriTextRangeA *r) {
+ const int size = r->afterLast - r->first;
+ if (size) {
+ return std::string(r->first, size);
+ }
+ return "";
+}
+
+bool parse_int(const UriTextRangeA *r, optional<uint16_t> * result) {
+ assert(result); // output
+ std::string int_string = copy_range(r);
+ if (!int_string.empty()) {
+ errno = 0;
+ unsigned long val = ::strtoul(int_string.c_str(), nullptr, 10);
+ if (errno == 0 && val < std::numeric_limits<uint16_t>::max() ) {
+ *result = std::experimental::make_optional<uint16_t>(val);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ // No value
+ *result = nullopt;
+ return true;
+}
+
+std::vector<std::string> copy_path(const UriPathSegmentA *ps) {
+ std::vector<std::string> result;
+ if (nullptr == ps)
+ return result;
+
+ for (; ps != 0; ps = ps->next) {
+ result.push_back(copy_range(&ps->text));
+ }
+
+ return result;
+}
+
+void parse_user_info(const UriTextRangeA *r, std::string * user, std::string * pass) {
+ // Output parameters
+ assert(user);
+ assert(pass);
+
+ std::string user_and_password = copy_range(r);
+ if (!user_and_password.empty()) {
+ const char * begin = user_and_password.c_str();
+ const char * colon_loc = strchr(begin, ':');
+ if (colon_loc) {
+ *user = std::string(begin, colon_loc - begin - 1);
+ *pass = colon_loc + 1;
+ } else {
+ *user = user_and_password;
+ }
+ }
+}
+
+
+std::vector<std::pair<std::string, std::string > > parse_query(const char *first, const char * afterLast) {
+ std::vector<std::pair<std::string, std::string > > result;
+ UriQueryListA * query;
+ int count;
+ int dissect_result = uriDissectQueryMallocExA(&query, &count, first, afterLast, false, URI_BR_DONT_TOUCH);
+ if (URI_SUCCESS == dissect_result) {
+ for (auto ps = query; ps != nullptr; ps = ps->next) {
+ std::string key = ps->key ? URI::encode(ps->key) : "";
+ std::string value = ps->value ? URI::encode(ps->value) : "";
+ result.push_back(std::make_pair(key, value));
+ }
+ uriFreeQueryListA(query);
+ }
+
+ return result;
+}
+
+
+optional<URI> URI::parse_from_string(const std::string &str)
+{
+ URI ret;
+ bool ok = true;
+
+ UriParserStateA state;
+ memset(&state, 0, sizeof(state));
+ UriUriA uu;
+
+ state.uri = &uu;
+ int parseResult = uriParseUriA(&state, str.c_str());
+ ok &= (parseResult == URI_SUCCESS);
+
+ if (ok) {
+ ret.scheme = copy_range(&uu.scheme);
+ ret.host = copy_range(&uu.hostText);
+ ok &= parse_int(&uu.portText, &ret.port);
+ ret.path = copy_path(uu.pathHead);
+ ret.query = parse_query(uu.query.first, uu.query.afterLast);
+ ret.fragment = copy_range(&uu.fragment);
+ parse_user_info(&uu.userInfo, &ret.user, &ret.pass);
+ uriFreeUriMembersA(&uu);
+ }
+ uriFreeUriMembersA(&uu);
+
+ if (ok) {
+ return std::experimental::make_optional(ret);
+ } else {
+ return nullopt;
+ }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+//
+// Getters and setters
+//
+///////////////////////////////////////////////////////////////////////////////
+
+std::string URI::str(bool encoded_output) const
+{
+ std::stringstream ss;
+ if (!scheme.empty()) ss << from_encoded(encoded_output, scheme) << "://";
+ if (!user.empty() || !pass.empty()) {
+ if (!user.empty()) ss << from_encoded(encoded_output, user);
+ if (!pass.empty()) ss << ":" << from_encoded(encoded_output, pass);
+ ss << "@";
+ }
+ if (has_authority()) ss << build_authority(encoded_output);
+ if (!path.empty()) ss << get_path(encoded_output);
+ if (!query.empty()) ss << "?" << get_query(encoded_output);
+ if (!fragment.empty()) ss << "#" << from_encoded(encoded_output, fragment);
+
+ return ss.str();
+}
+
+bool URI::has_authority() const
+{
+ return (!host.empty()) || (port);
+}
+
+std::string URI::build_authority(bool encoded_output) const
+{
+ std::stringstream ss;
+ ss << URI::from_encoded(encoded_output, host);
+ if (port)
+ {
+ ss << ":" << *port;
+ }
+ return ss.str();
+}
+
+
+std::string URI::get_path(bool encoded_output) const
+{
+ std::ostringstream out;
+ for (auto s: path) {
+ out << "/" << from_encoded(encoded_output, s);
+ }
+ return out.str();
+}
+
+std::vector<std::string> URI::get_path_elements(bool encoded_output) const
+{
+ std::vector<std::string> result;
+ for (auto path_elem: path) {
+ result.push_back(from_encoded(encoded_output, path_elem));
+ }
+
+ return result;
+}
+
+void URI::parse_path(bool input_encoded, const std::string &input_path)
+{
+ std::vector<std::string> split_path = split(input_path, '/');
+ for (auto s: split_path) {
+ path.push_back(to_encoded(input_encoded, s));
+ }
+}
+
+
+// Mostly copied and modified from uriparser2.c
+
+void URI::add_path(const std::string &p, bool encoded_input)
+{
+ path.push_back(to_encoded(encoded_input, p));
+}
+
+
+std::string URI::get_query(bool encoded_output) const {
+ bool first = true;
+ std::stringstream ss;
+ for (auto q: query) {
+ if (!first) {
+ ss << "&";
+ }
+ ss << from_encoded(encoded_output, q.first) << "=" << from_encoded(encoded_output, q.second);
+ first = false;
+ }
+
+ return ss.str();
+}
+
+std::vector< std::pair<std::string, std::string> > URI::get_query_elements(bool encoded_output) const
+{
+ std::vector< std::pair<std::string, std::string> > result;
+ for (auto q: query) {
+ auto key = from_encoded(encoded_output, q.first);
+ auto value = from_encoded(encoded_output, q.second);
+ result.push_back(std::make_pair(key, value));
+ }
+
+ return result;
+}
+
+
+void URI::set_query(const std::string &q) {
+ query = parse_query(q.c_str(), q.c_str() + q.size() + 1);
+}
+
+
+void URI::add_query(const std::string &name, const std::string & value, bool encoded_input)
+{
+ query.push_back(std::make_pair(to_encoded(encoded_input, name), to_encoded(encoded_input, value)));
+}
+
+void URI::remove_queries(const std::string &q_name, bool encoded_input)
+{
+ if (query.empty())
+ return;
+
+ // This is the one place we need to do decoded comparisons
+ std::string decoded_key = encoded_input ? decode(q_name) : q_name;
+
+ for (int i = query.size() - 1; i >= 0; i--) {
+ if (decode(query[i].first) == decoded_key) {
+ query.erase(query.begin() + i);
+ }
+ }
+}
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77515072/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.h
new file mode 100644
index 0000000..e6fbd78
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/uri.h
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef COMMON_HDFS_URI_H_
+#define COMMON_HDFS_URI_H_
+
+#include <iostream>
+#include <string>
+#include <optional.hpp>
+#include <vector>
+
+namespace hdfs
+{
+
+template <class T>
+using optional = std::experimental::optional<T>;
+
+class URI
+{
+ // These are stored in encoded form
+ std::string scheme;
+ std::string user;
+ std::string pass;
+ std::string host;
+ optional<uint16_t> port;
+ std::vector<std::string> path;
+ std::vector<std::pair<std::string,std::string> > query;
+ std::string fragment;
+
+ template <class T>
+ static T from_encoded(bool encoded_output, const T & input) {return encoded_output ? input : decode(input);}
+
+ template <class T>
+ static T to_encoded(bool encoded_input, const T & input) {return encoded_input ? input : encode(input);}
+
+ bool has_authority() const;
+ std::string build_authority(bool encoded_output) const;
+
+ std::string build_path(bool encoded_output) const;
+ void parse_path(bool input_encoded, const std::string &input_path);
+
+public:
+ // Parse a string into a URI. Returns nullopt if the URI is malformed.
+ static optional<URI> parse_from_string(const std::string &str);
+
+ static std::string encode (const std::string &input);
+ static std::string decode (const std::string &input);
+
+ std::string get_scheme(bool encoded_output=false) const
+ { return from_encoded(encoded_output,scheme); }
+
+ void set_scheme(const std::string &s, bool encoded_input=false)
+ { scheme = to_encoded(encoded_input,s); }
+
+ // empty if none.
+ std::string get_host(bool encoded_output=false) const
+ { return from_encoded(encoded_output,host); }
+
+ void set_host(const std::string& h, bool encoded_input=false)
+ { host = to_encoded(encoded_input,h); }
+
+ // -1 if the port is undefined.
+ optional<uint16_t> get_port() const
+ { return port; }
+
+ void set_port(uint16_t p)
+ { port = p; }
+
+ void clear_port()
+ { port = std::experimental::nullopt; }
+
+ std::string get_path(bool encoded_output=false) const;
+
+ std::vector<std::string> get_path_elements(bool encoded_output=false) const;
+
+ void set_path(const std::string &p, bool encoded_input=false) {
+ parse_path(encoded_input, p);
+ }
+
+ void add_path(const std::string &p, bool encoded_input=false);
+
+ std::string get_query(bool encoded_output=false) const;
+
+ std::vector< std::pair<std::string, std::string> > get_query_elements(bool encoded_output=false) const;
+
+ // Not that set_query must always pass in encoded strings
+ void set_query(const std::string &q);
+
+ // Adds a parameter onto the query; does not check if it already exists
+ // e.g. parseFromString("foo?bar=baz").addQuery("bing","bang")
+ // would leave "bar=baz&bing=bang" as the query
+ void add_query(const std::string &name, const std::string & value, bool encoded_input=false);
+
+ // Removes the query part if exists
+ // e.g. parseFromString("foo?bar=baz&bing=bang&bar=bong").removeQueries("bar")
+ // would leave bing=bang as the query
+ void remove_queries(const std::string &q_name, bool encoded_input=false);
+
+ std::string get_fragment(bool encoded_output=false) const
+ { return from_encoded(encoded_output, fragment); }
+
+ void set_fragment(const std::string &f, bool encoded_input=false)
+ { fragment = to_encoded(encoded_input,f); }
+
+ std::string str(bool encoded_output=true) const;
+};
+
+inline std::ostream& operator<<(std::ostream &out, const URI &uri)
+{ return out << uri.str(); }
+
+}
+
+#endif
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77515072/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
index 0ced667..0b28488 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
@@ -37,6 +37,9 @@ static const int kNamenodeProtocolVersion = 1;
using ::asio::ip::tcp;
+static constexpr uint16_t kDefaultPort = 8020;
+
+
/*****************************************************************************
* NAMENODE OPERATIONS
****************************************************************************/
@@ -148,7 +151,8 @@ const std::string get_effective_user_name(const std::string &user_name) {
FileSystemImpl::FileSystemImpl(IoService *&io_service, const std::string &user_name,
const Options &options)
- : io_service_(static_cast<IoServiceImpl *>(io_service)),
+ : options_(options),
+ io_service_(static_cast<IoServiceImpl *>(io_service)),
nn_(&io_service_->io_service(), options,
GetRandomClientName(), get_effective_user_name(user_name), kNamenodeProtocol,
kNamenodeProtocolVersion), client_name_(GetRandomClientName()),
@@ -175,7 +179,7 @@ FileSystemImpl::~FileSystemImpl() {
void FileSystemImpl::Connect(const std::string &server,
const std::string &service,
- const std::function<void(const Status &, FileSystem * fs)> &&handler) {
+ const std::function<void(const Status &, FileSystem * fs)> &handler) {
/* IoService::New can return nullptr */
if (!io_service_) {
handler (Status::Error("Null IoService"), this);
@@ -204,6 +208,48 @@ Status FileSystemImpl::Connect(const std::string &server, const std::string &ser
return s;
}
+void FileSystemImpl::ConnectToDefaultFs(const std::function<void(const Status &, FileSystem *)> &handler) {
+ std::string scheme = options_.defaultFS.get_scheme();
+ if (strcasecmp(scheme.c_str(), "hdfs") != 0) {
+ std::string error_message;
+ error_message += "defaultFS of [" + options_.defaultFS.str() + "] is not supported";
+ handler(Status::InvalidArgument(error_message.c_str()), nullptr);
+ return;
+ }
+
+ std::string host = options_.defaultFS.get_host();
+ if (host.empty()) {
+ handler(Status::InvalidArgument("defaultFS must specify a hostname"), nullptr);
+ return;
+ }
+
+ optional<uint16_t> port = options_.defaultFS.get_port();
+ if (!port) {
+ port = kDefaultPort;
+ }
+ std::string port_as_string = std::to_string(*port);
+
+ Connect(host, port_as_string, handler);
+}
+
+Status FileSystemImpl::ConnectToDefaultFs() {
+ auto stat = std::make_shared<std::promise<Status>>();
+ std::future<Status> future = stat->get_future();
+
+ auto callback = [stat](const Status &s, FileSystem *fs) {
+ (void)fs;
+ stat->set_value(s);
+ };
+
+ ConnectToDefaultFs(callback);
+
+ /* block until promise is set */
+ auto s = future.get();
+
+ return s;
+}
+
+
int FileSystemImpl::AddWorkerThread() {
auto service_task = [](IoService *service) { service->Run(); };
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77515072/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
index 726b8d8..b208a6c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
@@ -85,10 +85,14 @@ public:
/* attempt to connect to namenode, return bad status on failure */
void Connect(const std::string &server, const std::string &service,
- const std::function<void(const Status &, FileSystem *)> &&handler) override;
+ const std::function<void(const Status &, FileSystem *)> &handler) override;
/* attempt to connect to namenode, return bad status on failure */
Status Connect(const std::string &server, const std::string &service) override;
+ /* Connect to the NN indicated in options.defaultFs */
+ virtual void ConnectToDefaultFs(
+ const std::function<void(const Status &, FileSystem *)> &handler) override;
+ virtual Status ConnectToDefaultFs() override;
virtual void Open(const std::string &path,
const std::function<void(const Status &, FileHandle *)>
@@ -105,6 +109,7 @@ public:
private:
+ const Options options_;
/**
* The IoService must be the first member variable to ensure that it gets
* destroyed last. This allows other members to dequeue things from the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77515072/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
index 8aadcdb..a0b3774 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
@@ -55,6 +55,10 @@ function(add_memcheck_test name binary)
endfunction(add_memcheck_test)
+add_executable(uri_test uri_test.cc)
+target_link_libraries(uri_test common gmock_main ${CMAKE_THREAD_LIBS_INIT})
+add_memcheck_test(uri uri_test)
+
add_executable(remote_block_reader_test remote_block_reader_test.cc)
target_link_libraries(remote_block_reader_test test_common reader proto common connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT})
add_memcheck_test(remote_block_reader remote_block_reader_test)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77515072/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc
index 7d2a43c..4d49728 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc
@@ -483,6 +483,43 @@ TEST(ConfigurationTest, TestBoolConversions) {
}
}
+TEST(ConfigurationTest, TestUriConversions) {
+ /* No defaults */
+ {
+ std::stringstream stream;
+ simpleConfigStream(stream, "key1", "hdfs:///");
+ optional<Configuration> config = ConfigurationLoader().Load<Configuration>(stream.str());
+ EXPECT_TRUE(config && "Parse single value");
+ optional<URI> value = config->GetUri("key1");
+ EXPECT_TRUE((bool)value);
+ EXPECT_EQ("hdfs:///", value->str());
+ EXPECT_FALSE(config->GetUri("key2"));
+ }
+
+ {
+ optional<Configuration> config = simpleConfig("key1", "hdfs:///");
+ EXPECT_EQ("hdfs:///", config->GetUriWithDefault("key1", "http:///").str());
+ }
+ {
+ optional<Configuration> config = simpleConfig("key1", " hdfs:/// ");
+ EXPECT_EQ("hdfs:///", config->GetUriWithDefault("key1", "http:///").str());
+ }
+ {
+ optional<Configuration> config = simpleConfig("key1", "");
+ EXPECT_EQ("", config->GetUriWithDefault("key1", "http:///").str());
+ }
+ {
+ optional<Configuration> config = simpleConfig("key1", "%%"); // invalid URI
+ EXPECT_EQ("http:///", config->GetUriWithDefault("key1", "http:///").str());
+ }
+ {
+ optional<Configuration> config = simpleConfig("key2", "hdfs:///");
+ EXPECT_EQ("http:///", config->GetUriWithDefault("key1", "http:///").str());
+ }
+}
+
+
+
int main(int argc, char *argv[]) {
/*
* The following line must be executed to initialize Google Mock
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77515072/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/uri_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/uri_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/uri_test.cc
new file mode 100644
index 0000000..1dd870a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/uri_test.cc
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common/uri.h"
+#include <gmock/gmock.h>
+
+using ::testing::_;
+
+using namespace hdfs;
+
+TEST(UriTest, TestDegenerateInputs) {
+ /* Empty input */
+ {
+ optional<URI> uri = URI::parse_from_string("");
+ EXPECT_TRUE(uri && "Empty input");
+ }
+
+ /* Invalid encoding */
+ {
+ optional<URI> uri = URI::parse_from_string("%%");
+ EXPECT_FALSE(uri && "Bad input");
+ }
+
+ /* Invalid port */
+ {
+ optional<URI> uri = URI::parse_from_string("hdfs://nn:foo/");
+ EXPECT_FALSE(uri && "Bad port");
+ }
+
+ /* Negative port */
+ {
+ optional<URI> uri = URI::parse_from_string("hdfs://nn:-100/");
+ EXPECT_FALSE(uri && "Negative port");
+ }
+
+ /* Empty paths */
+ {
+ optional<URI> uri = URI::parse_from_string("hdfs://////");
+ EXPECT_TRUE(uri && "Empty paths");
+ }
+
+}
+
+
+TEST(UriTest, TestNominalInputs) {
+ /* Simple input */
+ {
+ optional<URI> uri = URI::parse_from_string("hdfs:///foo");
+ ASSERT_TRUE(uri && "Parsed");
+ EXPECT_EQ("hdfs", uri->get_scheme());
+ EXPECT_EQ("", uri->get_host());
+ EXPECT_EQ(0, uri->get_port().value_or(0));
+ EXPECT_EQ("/foo", uri->get_path());
+ EXPECT_EQ("", uri->get_fragment());
+ EXPECT_EQ("", uri->get_query());
+ }
+
+ /* With authority */
+ {
+ optional<URI> uri = URI::parse_from_string("hdfs://host:100/foo");
+ ASSERT_TRUE(uri && "Parsed");
+ EXPECT_EQ("hdfs", uri->get_scheme());
+ EXPECT_EQ("host", uri->get_host());
+ EXPECT_EQ(100, uri->get_port().value_or(0));
+ EXPECT_EQ("/foo", uri->get_path());
+ EXPECT_EQ("", uri->get_fragment());
+ EXPECT_EQ("", uri->get_query());
+ }
+
+ /* No scheme */
+ {
+ optional<URI> uri = URI::parse_from_string("/foo");
+ ASSERT_TRUE(uri && "Parsed");
+ EXPECT_EQ("", uri->get_scheme());
+ EXPECT_EQ("", uri->get_host());
+ EXPECT_EQ(0, uri->get_port().value_or(0));
+ EXPECT_EQ("/foo", uri->get_path());
+ EXPECT_EQ("", uri->get_fragment());
+ EXPECT_EQ("", uri->get_query());
+ }
+
+ /* All fields */
+ {
+ optional<URI> uri = URI::parse_from_string("hdfs://nn:8020/path/to/data?a=b&c=d#fragment");
+ ASSERT_TRUE(uri && "Parsed");
+ EXPECT_EQ("hdfs", uri->get_scheme());
+ EXPECT_EQ("nn", uri->get_host());
+ EXPECT_EQ(8020, uri->get_port().value_or(0));
+ EXPECT_EQ("/path/to/data", uri->get_path());
+ EXPECT_EQ("a=b&c=d", uri->get_query());
+ EXPECT_EQ(3, uri->get_path_elements().size());
+ EXPECT_EQ("path", uri->get_path_elements()[0]);
+ EXPECT_EQ("to", uri->get_path_elements()[1]);
+ EXPECT_EQ("data", uri->get_path_elements()[2]);
+ EXPECT_EQ(2, uri->get_query_elements().size());
+ EXPECT_EQ("a", uri->get_query_elements()[0].first);
+ EXPECT_EQ("b", uri->get_query_elements()[0].second);
+ EXPECT_EQ("c", uri->get_query_elements()[1].first);
+ EXPECT_EQ("d", uri->get_query_elements()[1].second);
+ EXPECT_EQ("fragment", uri->get_fragment());
+ }
+}
+
+TEST(UriTest, TestEncodedInputs) {
+ // Note that scheme and port cannot be uri-encoded
+
+ /* Encoded input */
+ {
+ optional<URI> uri = URI::parse_from_string("S://%5E:1/+%5E%20?%5E=%5E#%5E");
+ ASSERT_TRUE(uri && "Parsed");
+ EXPECT_EQ("S", uri->get_scheme());
+ EXPECT_EQ("^", uri->get_host());
+ EXPECT_EQ(1, uri->get_port().value_or(0));
+ EXPECT_EQ("/ ^ ", uri->get_path());
+ EXPECT_EQ("^", uri->get_fragment());
+ EXPECT_EQ("^=^", uri->get_query());
+ }
+
+ /* Lowercase */
+ {
+ optional<URI> uri = URI::parse_from_string("S://%5e:1/+%5e%20?%5e=%5e#%5e");
+ ASSERT_TRUE(uri && "Parsed");
+ EXPECT_EQ("S", uri->get_scheme());
+ EXPECT_EQ("^", uri->get_host());
+ EXPECT_EQ(1, uri->get_port().value_or(0));
+ EXPECT_EQ("/ ^ ", uri->get_path());
+ EXPECT_EQ("^", uri->get_fragment());
+ EXPECT_EQ("^=^", uri->get_query());
+ }
+}
+
+TEST(UriTest, TestDecodedInputsAndOutputs) {
+ /* All fields non-encoded and shouldn't be interpreted */
+ {
+ optional<URI> uri = URI::parse_from_string("S://%25/%25+?%25=%25#%25");
+ ASSERT_TRUE(uri && "Parsed");
+ EXPECT_EQ("S", uri->get_scheme());
+ EXPECT_EQ("%", uri->get_host());
+ EXPECT_EQ(0, uri->get_port().value_or(0));
+ EXPECT_EQ("/% ", uri->get_path());
+ EXPECT_EQ("%", uri->get_fragment());
+ EXPECT_EQ("%=%", uri->get_query());
+ }
+
+ /* All fields encode fields on their way out */
+ {
+ optional<URI> uri = URI::parse_from_string("S://%25/%25+?%25=%25#%25");
+ ASSERT_TRUE(uri && "Parsed");
+ EXPECT_EQ("S", uri->get_scheme(true));
+ EXPECT_EQ("%25", uri->get_host(true));
+ EXPECT_EQ(0, uri->get_port().value_or(0));
+ EXPECT_EQ("/%25+", uri->get_path(true));
+ EXPECT_EQ("%25", uri->get_fragment(true));
+ EXPECT_EQ("%25=%25", uri->get_query(true));
+ }
+
+}
+
+TEST(UriTest, TestSetters) {
+
+ /* Non-encoded inputs */
+ {
+ URI uri;
+ uri.set_scheme("S");
+ uri.set_host("%");
+ uri.set_port(100);
+ uri.set_path("%/%/%");
+ uri.set_fragment("%");
+ uri.set_query("%25=%25"); //set_query must always be encoded
+ EXPECT_EQ("S://%25:100/%25/%25/%25?%25=%25#%25", uri.str());
+ }
+
+ /* Incremental adders, non-encoded */
+ {
+ URI uri;
+ uri.set_scheme("S");
+ uri.set_host("%");
+ uri.set_port(100);
+ uri.set_fragment("%");
+ EXPECT_EQ("S://%25:100#%25", uri.str());
+
+ uri.add_path("%");
+ uri.add_query("%", "%");
+ EXPECT_EQ("S://%25:100/%25?%25=%25#%25", uri.str());
+
+ uri.add_path("%");
+ uri.add_query("%", "%");
+ EXPECT_EQ("S://%25:100/%25/%25?%25=%25&%25=%25#%25", uri.str());
+ }
+
+ /* Encoded inputs */
+ {
+ URI uri;
+ uri.set_scheme("S", true);
+ uri.set_host("%25", true);
+ uri.set_port(100);
+ uri.set_path("%25/%25/%25", true);
+ uri.set_fragment("%25", true);
+ uri.set_query("%25=%25"); //set_query must always be encoded
+ EXPECT_EQ("S://%25:100/%25/%25/%25?%25=%25#%25", uri.str());
+ }
+
+ /* Incremental adders, encoded */
+ {
+ URI uri;
+ uri.set_scheme("S", true);
+ uri.set_host("%25", true);
+ uri.set_port(100);
+ uri.set_fragment("%25", true);
+ EXPECT_EQ("S://%25:100#%25", uri.str());
+
+ uri.add_path("%25", true);
+ uri.add_query("%25", "%25", true);
+ EXPECT_EQ("S://%25:100/%25?%25=%25#%25", uri.str());
+
+ uri.add_path("%25", true);
+ uri.add_query("%25", "%25", true);
+ EXPECT_EQ("S://%25:100/%25/%25?%25=%25&%25=%25#%25", uri.str());
+ }
+
+}
+
+int main(int argc, char *argv[]) {
+ /*
+ * The following line must be executed to initialize Google Mock
+ * (and Google Test) before running the tests.
+ */
+ ::testing::InitGoogleMock(&argc, argv);
+ return RUN_ALL_TESTS();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77515072/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/third_party/uriparser2/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/third_party/uriparser2/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/third_party/uriparser2/CMakeLists.txt
index 644cf3a..d752be5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/third_party/uriparser2/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/third_party/uriparser2/CMakeLists.txt
@@ -22,5 +22,5 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-attributes")
add_library(uriparser2_obj OBJECT uriparser2/uriparser2.c uriparser2/uriparser/UriParse.c uriparser2/uriparser/UriParseBase.c
- uriparser2/uriparser/UriCommon.c uriparser2/uriparser/UriIp4Base.c uriparser2/uriparser/UriIp4.c)
+ uriparser2/uriparser/UriCommon.c uriparser2/uriparser/UriIp4Base.c uriparser2/uriparser/UriIp4.c uriparser2/uriparser/UriEscape.c uriparser2/uriparser/UriQuery.c)
add_library(uriparser2 $<TARGET_OBJECTS:uriparser2_obj>)