You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by an...@apache.org on 2018/02/14 22:04:17 UTC
mesos git commit: Windows: Enabled `CurlFetcherPlugin` and
`HadoopFetcherPlugin`.
Repository: mesos
Updated Branches:
refs/heads/master e36e3b998 -> 1e7120833
Windows: Enabled `CurlFetcherPlugin` and `HadoopFetcherPlugin`.
Replaced checking for `/` with `path::absolute`, and fixed `argv[0]`
assumption when invoking `hadoop`.
Ported the HDFS mock script from Shell to Batch to emulate the `hadoop`
command.
Enabled the plugins and associated tests.
Review: https://reviews.apache.org/r/65617/
Review: https://reviews.apache.org/r/65618/
Review: https://reviews.apache.org/r/65619/
Review: https://reviews.apache.org/r/65620/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/1e712083
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/1e712083
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/1e712083
Branch: refs/heads/master
Commit: 1e71208338fc525d42cd8dd0ffad5a0841e40a92
Parents: e36e3b9
Author: Jeff Coffler <je...@taltos.com>
Authored: Wed Feb 14 12:33:20 2018 -0800
Committer: Andrew Schwartzmeyer <an...@schwartzmeyer.com>
Committed: Wed Feb 14 14:01:51 2018 -0800
----------------------------------------------------------------------
src/CMakeLists.txt | 6 +++---
src/hdfs/hdfs.cpp | 4 ++--
src/tests/uri_fetcher_tests.cpp | 35 ++++++++++++++++++++---------------
src/uri/fetcher.cpp | 4 ++--
src/uri/fetcher.hpp | 6 +++---
5 files changed, 30 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/1e712083/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 21fb47e..0c13503 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -373,12 +373,12 @@ set(URI_SRC
uri/fetcher.cpp
uri/utils.cpp
uri/fetchers/copy.cpp
- uri/fetchers/curl.cpp)
+ uri/fetchers/curl.cpp
+ uri/fetchers/hadoop.cpp)
if (NOT WIN32)
list(APPEND URI_SRC
- uri/fetchers/docker.cpp
- uri/fetchers/hadoop.cpp)
+ uri/fetchers/docker.cpp)
endif ()
set(USAGE_SRC
http://git-wip-us.apache.org/repos/asf/mesos/blob/1e712083/src/hdfs/hdfs.cpp
----------------------------------------------------------------------
diff --git a/src/hdfs/hdfs.cpp b/src/hdfs/hdfs.cpp
index f9fc1cbb5..726925f 100644
--- a/src/hdfs/hdfs.cpp
+++ b/src/hdfs/hdfs.cpp
@@ -143,7 +143,7 @@ Try<Owned<HDFS>> HDFS::create(const Option<string>& _hadoop)
static string normalize(const string& hdfsPath)
{
if (strings::contains(hdfsPath, "://") || // A URI or a malformed path.
- strings::startsWith(hdfsPath, "/")) { // Already an absolute path.
+ path::absolute(hdfsPath)) { // Already an absolute path.
return hdfsPath;
}
@@ -316,7 +316,7 @@ Future<Nothing> HDFS::copyToLocal(const string& from, const string& to)
{
Try<Subprocess> s = subprocess(
hadoop,
- {"hadoop", "fs", "-copyToLocal", normalize(from), to},
+ {hadoop, "fs", "-copyToLocal", normalize(from), to},
Subprocess::PATH(os::DEV_NULL),
Subprocess::PIPE(),
Subprocess::PIPE());
http://git-wip-us.apache.org/repos/asf/mesos/blob/1e712083/src/tests/uri_fetcher_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/uri_fetcher_tests.cpp b/src/tests/uri_fetcher_tests.cpp
index 0acb854..816d8ec 100644
--- a/src/tests/uri_fetcher_tests.cpp
+++ b/src/tests/uri_fetcher_tests.cpp
@@ -31,6 +31,7 @@
#include <stout/os/getcwd.hpp>
#include <stout/os/ls.hpp>
#include <stout/os/write.hpp>
+#include <stout/uri.hpp>
#include <stout/tests/utils.hpp>
@@ -158,12 +159,17 @@ public:
// Create a fake hadoop command line tool. It emulates the hadoop
// client's logic while operating on the local filesystem.
+#ifndef __WINDOWS__
hadoop = path::join(os::getcwd(), "hadoop");
+#else
+ hadoop = path::join(os::getcwd(), "hadoop.bat");
+#endif // __WINDOWS__
// The script emulating 'hadoop fs -copyToLocal <from> <to>'.
// NOTE: We emulate a version call here which is exercised when
// creating the HDFS client. But, we don't expect any other
// command to be called.
+#ifndef __WINDOWS__
ASSERT_SOME(os::write(
hadoop,
"#!/bin/sh\n"
@@ -178,12 +184,23 @@ public:
"fi\n"
"cp $3 $4\n"));
- // Make sure the script has execution permission. On Windows, we always
- // have permission.
-#ifndef __WINDOWS__
+ // Make sure the script has execution permission.
ASSERT_SOME(os::chmod(
hadoop,
S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH));
+#else
+ // NOTE: This is a `.bat` file instead of PowerShell so that it can be
+ // directly executed. Windows "knows" how to launch files ending in `.bat`,
+ // similar to the shebang logic for POSIX. However, this does not extend to
+ // PowerShell's `.ps1` scripts.
+ ASSERT_SOME(os::write(
+ hadoop,
+ "if \"%1\" == \"version\" (exit 0)\n"
+ "if NOT \"%1\" == \"fs\" (exit 1)\n"
+ "if NOT \"%2\" == \"-copyToLocal\" (exit 1)\n"
+ "copy %3 %4\n"));
+
+ // Windows has no notion of "execution permission".
#endif // __WINDOWS__
}
@@ -192,9 +209,6 @@ protected:
};
-// TODO(hausdorff): Will not compile until HDFS is supported on Windows. See
-// MESOS-5460.
-#ifndef __WINDOWS__
TEST_F(HadoopFetcherPluginTest, FetchExistingFile)
{
string file = path::join(os::getcwd(), "file");
@@ -215,12 +229,8 @@ TEST_F(HadoopFetcherPluginTest, FetchExistingFile)
EXPECT_SOME_EQ("abc", os::read(path::join(dir, "file")));
}
-#endif // __WINDOWS__
-// TODO(hausdorff): Will not compile until HDFS is supported on Windows. See
-// MESOS-5460.
-#ifndef __WINDOWS__
TEST_F(HadoopFetcherPluginTest, FetchNonExistingFile)
{
URI uri = uri::hdfs(path::join(os::getcwd(), "non-exist"));
@@ -235,12 +245,8 @@ TEST_F(HadoopFetcherPluginTest, FetchNonExistingFile)
AWAIT_FAILED(fetcher.get()->fetch(uri, dir));
}
-#endif // __WINDOWS__
-// TODO(hausdorff): Will not compile until HDFS is supported on Windows. See
-// MESOS-5460.
-#ifndef __WINDOWS__
// This test verifies invoking 'fetch' by plugin name.
TEST_F(HadoopFetcherPluginTest, InvokeFetchByName)
{
@@ -262,7 +268,6 @@ TEST_F(HadoopFetcherPluginTest, InvokeFetchByName)
EXPECT_SOME_EQ("abc", os::read(path::join(dir, "file")));
}
-#endif // __WINDOWS__
// TODO(jieyu): Expose this constant so that other docker related
http://git-wip-us.apache.org/repos/asf/mesos/blob/1e712083/src/uri/fetcher.cpp
----------------------------------------------------------------------
diff --git a/src/uri/fetcher.cpp b/src/uri/fetcher.cpp
index 13c2d54..489c7ce 100644
--- a/src/uri/fetcher.cpp
+++ b/src/uri/fetcher.cpp
@@ -50,10 +50,10 @@ Try<Owned<Fetcher>> create(const Option<Flags>& _flags)
[flags]() { return CurlFetcherPlugin::create(flags); }},
{CopyFetcherPlugin::NAME,
[flags]() { return CopyFetcherPlugin::create(flags); }},
-#ifndef __WINDOWS__
- // TODO(dpravat): Enable `Hadoop` and `Docker` plugins. See MESOS-5473.
{HadoopFetcherPlugin::NAME,
[flags]() { return HadoopFetcherPlugin::create(flags); }},
+#ifndef __WINDOWS__
+ // TODO(coffler): Enable Docker plugin. See MESOS-8570.
{DockerFetcherPlugin::NAME,
[flags]() { return DockerFetcherPlugin::create(flags); }},
#endif // __WINDOWS__
http://git-wip-us.apache.org/repos/asf/mesos/blob/1e712083/src/uri/fetcher.hpp
----------------------------------------------------------------------
diff --git a/src/uri/fetcher.hpp b/src/uri/fetcher.hpp
index 2b2b14e..fbf64c6 100644
--- a/src/uri/fetcher.hpp
+++ b/src/uri/fetcher.hpp
@@ -39,11 +39,11 @@ namespace fetcher {
*/
class Flags :
public virtual CopyFetcherPlugin::Flags,
+ public virtual CurlFetcherPlugin::Flags,
#ifdef __WINDOWS__
- // TODO(dpravat): Add support for Hadoop and Docker plugins. See MESOS-5473.
- public virtual CurlFetcherPlugin::Flags {};
+ public virtual HadoopFetcherPlugin::Flags {};
#else
- public virtual CurlFetcherPlugin::Flags,
+ // TODO(coffler): Add support for Docker plugins. See MESOS-8570.
public virtual HadoopFetcherPlugin::Flags,
public virtual DockerFetcherPlugin::Flags {};
#endif // __WINDOWS__