You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by be...@apache.org on 2016/01/22 11:37:25 UTC
mesos git commit: Fixed support for non-HDFS URIs by Hadoop client.
Repository: mesos
Updated Branches:
refs/heads/master 6195e783d -> 6afd1b682
Fixed support for non-HDFS URIs by Hadoop client.
Review: https://reviews.apache.org/r/42246/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/6afd1b68
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/6afd1b68
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/6afd1b68
Branch: refs/heads/master
Commit: 6afd1b682df9809a9113ab9f61caf26b8c6a02d9
Parents: 6195e78
Author: Bernd Mathiske <be...@mesosphere.io>
Authored: Fri Jan 22 11:36:12 2016 +0100
Committer: Bernd Mathiske <be...@mesosphere.io>
Committed: Fri Jan 22 11:36:37 2016 +0100
----------------------------------------------------------------------
src/hdfs/hdfs.cpp | 36 ++++++++++++++++++++----------------
src/hdfs/hdfs.hpp | 4 ----
2 files changed, 20 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/6afd1b68/src/hdfs/hdfs.cpp
----------------------------------------------------------------------
diff --git a/src/hdfs/hdfs.cpp b/src/hdfs/hdfs.cpp
index 9c91d1f..bebba8b 100644
--- a/src/hdfs/hdfs.cpp
+++ b/src/hdfs/hdfs.cpp
@@ -122,11 +122,26 @@ Try<Owned<HDFS>> HDFS::create(const Option<string>& _hadoop)
}
+// An HDFS client path must be either a full URI or an absolute path. If it is
+// a relative path, prepend "/" to make it absolute. (Note that all URI schemes
+// supported by the HDFS client contain "://" whereas file paths never do.)
+static string normalize(const string& hdfsPath)
+{
+ if (strings::contains(hdfsPath, "://") || // A URI or a malformed path.
+ strings::startsWith(hdfsPath, "/")) { // Already an absolute path.
+ return hdfsPath;
+ }
+
+ // A relative, non-URI file path. Prepend "/".
+ return path::join("", hdfsPath);
+}
+
+
Future<bool> HDFS::exists(const string& path)
{
Try<Subprocess> s = subprocess(
hadoop,
- {"hadoop", "fs", "-test", "-e", absolutePath(path)},
+ {"hadoop", "fs", "-test", "-e", normalize(path)},
Subprocess::PATH("/dev/null"),
Subprocess::PIPE(),
Subprocess::PIPE());
@@ -161,7 +176,7 @@ Future<bool> HDFS::exists(const string& path)
Future<Bytes> HDFS::du(const string& _path)
{
- const string path = absolutePath(_path);
+ const string path = normalize(_path);
Try<Subprocess> s = subprocess(
hadoop,
@@ -218,7 +233,7 @@ Future<Nothing> HDFS::rm(const string& path)
{
Try<Subprocess> s = subprocess(
hadoop,
- {"hadoop", "fs", "-rm", absolutePath(path)},
+ {"hadoop", "fs", "-rm", normalize(path)},
Subprocess::PATH("/dev/null"),
Subprocess::PIPE(),
Subprocess::PIPE());
@@ -254,7 +269,7 @@ Future<Nothing> HDFS::copyFromLocal(const string& from, const string& to)
Try<Subprocess> s = subprocess(
hadoop,
- {"hadoop", "fs", "-copyFromLocal", from, absolutePath(to)},
+ {"hadoop", "fs", "-copyFromLocal", from, normalize(to)},
Subprocess::PATH("/dev/null"),
Subprocess::PIPE(),
Subprocess::PIPE());
@@ -286,7 +301,7 @@ Future<Nothing> HDFS::copyToLocal(const string& from, const string& to)
{
Try<Subprocess> s = subprocess(
hadoop,
- {"hadoop", "fs", "-copyToLocal", absolutePath(from), to},
+ {"hadoop", "fs", "-copyToLocal", normalize(from), to},
Subprocess::PATH("/dev/null"),
Subprocess::PIPE(),
Subprocess::PIPE());
@@ -312,14 +327,3 @@ Future<Nothing> HDFS::copyToLocal(const string& from, const string& to)
return Nothing();
});
}
-
-
-string HDFS::absolutePath(const string& hdfsPath)
-{
- if (strings::startsWith(hdfsPath, "hdfs://") ||
- strings::startsWith(hdfsPath, "/")) {
- return hdfsPath;
- }
-
- return path::join("", hdfsPath);
-}
http://git-wip-us.apache.org/repos/asf/mesos/blob/6afd1b68/src/hdfs/hdfs.hpp
----------------------------------------------------------------------
diff --git a/src/hdfs/hdfs.hpp b/src/hdfs/hdfs.hpp
index abdb9b9..716d13f 100644
--- a/src/hdfs/hdfs.hpp
+++ b/src/hdfs/hdfs.hpp
@@ -65,10 +65,6 @@ private:
explicit HDFS(const std::string& _hadoop)
: hadoop(_hadoop) {}
- // Normalize an HDFS path such that it is either an absolute path or
- // a full hdfs:// URL.
- std::string absolutePath(const std::string& hdfsPath);
-
const std::string hadoop;
};