You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by an...@apache.org on 2018/05/21 18:21:53 UTC
[1/2] mesos git commit: Windows: Removed guards now that `os::shell`
exists.
Repository: mesos
Updated Branches:
refs/heads/master 9fa72c549 -> 04263336e
Windows: Removed guards now that `os::shell` exists.
Review: https://reviews.apache.org/r/67189
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/04263336
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/04263336
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/04263336
Branch: refs/heads/master
Commit: 04263336e4cc9880f6733c2f4e113ed608e92532
Parents: 7f115f4
Author: Andrew Schwartzmeyer <an...@schwartzmeyer.com>
Authored: Thu May 17 10:53:30 2018 -0700
Committer: Andrew Schwartzmeyer <an...@schwartzmeyer.com>
Committed: Mon May 21 10:45:53 2018 -0700
----------------------------------------------------------------------
src/master/main.cpp | 6 ------
src/slave/main.cpp | 12 ------------
src/tests/fetcher_tests.cpp | 27 +++++----------------------
3 files changed, 5 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/04263336/src/master/main.cpp
----------------------------------------------------------------------
diff --git a/src/master/main.cpp b/src/master/main.cpp
index 3f5b354..3eb1f89 100644
--- a/src/master/main.cpp
+++ b/src/master/main.cpp
@@ -200,7 +200,6 @@ int main(int argc, char** argv)
}
if (flags.ip_discovery_command.isSome()) {
-#ifndef __WINDOWS__
Try<string> ipAddress = os::shell(flags.ip_discovery_command.get());
if (ipAddress.isError()) {
@@ -208,11 +207,6 @@ int main(int argc, char** argv)
}
os::setenv("LIBPROCESS_IP", strings::trim(ipAddress.get()));
-#else
- // TODO(andschwa): Support this when `os::shell` is enabled.
- EXIT(EXIT_FAILURE)
- << "The `--ip.discovery_command` is not yet supported on Windows";
-#endif // __WINDOWS__
} else if (flags.ip.isSome()) {
os::setenv("LIBPROCESS_IP", flags.ip.get());
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/04263336/src/slave/main.cpp
----------------------------------------------------------------------
diff --git a/src/slave/main.cpp b/src/slave/main.cpp
index ca35668..6461253 100644
--- a/src/slave/main.cpp
+++ b/src/slave/main.cpp
@@ -320,7 +320,6 @@ int main(int argc, char** argv)
}
if (flags.ip_discovery_command.isSome()) {
-#ifndef __WINDOWS__
Try<string> ipAddress = os::shell(flags.ip_discovery_command.get());
if (ipAddress.isError()) {
@@ -328,28 +327,17 @@ int main(int argc, char** argv)
}
os::setenv("LIBPROCESS_IP", strings::trim(ipAddress.get()));
-#else
- // TODO(andschwa): Support this when `os::shell` is enabled.
- EXIT(EXIT_FAILURE)
- << "The `--ip_discovery_command` is not yet supported on Windows";
-#endif // __WINDOWS__
} else if (flags.ip.isSome()) {
os::setenv("LIBPROCESS_IP", flags.ip.get());
}
if (flags.ip6_discovery_command.isSome()) {
-#ifndef __WINDOWS__
Try<string> ip6Address = os::shell(flags.ip6_discovery_command.get());
if (ip6Address.isError()) {
EXIT(EXIT_FAILURE) << ip6Address.error();
}
os::setenv("LIBPROCESS_IP6", strings::trim(ip6Address.get()));
-#else
- // TODO(andschwa): Support this when `os::shell` is enabled.
- EXIT(EXIT_FAILURE)
- << "The `--ip6_discovery_command` is not yet supported on Windows";
-#endif // __WINDOWS__
} else if (flags.ip6.isSome()) {
os::setenv("LIBPROCESS_IP6", flags.ip6.get());
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/04263336/src/tests/fetcher_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/fetcher_tests.cpp b/src/tests/fetcher_tests.cpp
index 311be6a..95359ec 100644
--- a/src/tests/fetcher_tests.cpp
+++ b/src/tests/fetcher_tests.cpp
@@ -756,10 +756,8 @@ TEST_F_TEMP_DISABLED_ON_WINDOWS(FetcherTest, NoExtractExecutable)
}
-// NOTE: This is disabled on Windows because `os::shell()` is deleted.
-// Also, permissions handling needs to be worked out, see MESOS-3176.
-#ifndef __WINDOWS__
-TEST_F(FetcherTest, ExtractNotExecutable)
+// NOTE: Permissions handling needs to be worked out, see MESOS-3176.
+TEST_F_TEMP_DISABLED_ON_WINDOWS(FetcherTest, ExtractNotExecutable)
{
// First construct a temporary file that can be fetched and archived with tar
// gzip.
@@ -811,17 +809,14 @@ TEST_F(FetcherTest, ExtractNotExecutable)
verifyMetrics(1, 0);
}
-#endif // __WINDOWS__
// Tests extracting tar file with extension .tar.
-// NOTE: This is disabled on Windows because `os::shell()` is deleted.
//
// Won't be supported on Windows for now; long term thoughts are to perhaps
// use a code library to provide 'tar' functionality programmatically,
// see MESOS-8064.
-#ifndef __WINDOWS__
-TEST_F(FetcherTest, ExtractTar)
+TEST_F_TEMP_DISABLED_ON_WINDOWS(FetcherTest, ExtractTar)
{
// First construct a temporary file that can be fetched and archived with
// tar.
@@ -864,16 +859,12 @@ TEST_F(FetcherTest, ExtractTar)
verifyMetrics(1, 0);
}
-#endif // __WINDOWS__
-// NOTE: This is disabled on Windows because `os::shell()` is deleted.
-//
// Won't be supported on Windows for now; long term thoughts are to perhaps
// use a code library to provide 'gzip' functionality programmatically,
// see MESOS-8064.
-#ifndef __WINDOWS__
-TEST_F(FetcherTest, ExtractGzipFile)
+TEST_F_TEMP_DISABLED_ON_WINDOWS(FetcherTest, ExtractGzipFile)
{
// First construct a temporary file that can be fetched and archived with
// gzip.
@@ -911,7 +902,6 @@ TEST_F(FetcherTest, ExtractGzipFile)
verifyMetrics(1, 0);
}
-#endif // __WINDOWS__
TEST_F(FetcherTest, UNZIP_ExtractFile)
@@ -1133,8 +1123,6 @@ TEST_F(FetcherTest, UseCustomOutputFile)
}
-// NOTE: This is disabled on Windows because `os::shell()` is deleted.
-#ifndef __WINDOWS__
TEST_F_TEMP_DISABLED_ON_WINDOWS(FetcherTest, CustomGzipOutputFile)
{
// First construct a temporary file that can be fetched.
@@ -1174,7 +1162,6 @@ TEST_F_TEMP_DISABLED_ON_WINDOWS(FetcherTest, CustomGzipOutputFile)
verifyMetrics(1, 0);
}
-#endif // __WINDOWS__
// TODO(hausdorff): `os::chmod` does not exist on Windows.
@@ -1275,10 +1262,7 @@ TEST_F(FetcherTest, HdfsURI)
// agent towards the fetcher. By supplying an invalid SSL setup, we
// force the fetcher to fail if the parent process does not filter
// them out.
-//
-// NOTE: This is disabled on Windows because `os::shell()` is deleted.
-#ifndef __WINDOWS__
-TEST_F(FetcherTest, SSLEnvironmentSpillover)
+TEST_F_TEMP_DISABLED_ON_WINDOWS(FetcherTest, SSLEnvironmentSpillover)
{
// Patch some critical libprocess environment variables into the
// parent process of the mesos-fetcher. We expect this test to fail
@@ -1332,7 +1316,6 @@ TEST_F(FetcherTest, SSLEnvironmentSpillover)
os::setenv("LIBPROCESS_SSL_KEY_FILE", key);
}
}
-#endif // __WINDOWS__
} // namespace tests {
} // namespace internal {
[2/2] mesos git commit: Windows: Implemented `os::shell` and enabled
tests.
Posted by an...@apache.org.
Windows: Implemented `os::shell` and enabled tests.
Once implemented, the tests had to be fixed to use Windows commands,
and a test that ensures `os::shell` doesn't hang was added. The
implementation does not reuse `os::spawn` because it must read all the
child process's piped output before waiting on it to exit, otherwise
it will deadlock.
Review: https://reviews.apache.org/r/67188
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/7f115f47
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/7f115f47
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/7f115f47
Branch: refs/heads/master
Commit: 7f115f470e0d9bc8bbb68a2ca16f73b6e318035c
Parents: 9fa72c5
Author: Andrew Schwartzmeyer <an...@schwartzmeyer.com>
Authored: Thu May 17 10:50:05 2018 -0700
Committer: Andrew Schwartzmeyer <an...@schwartzmeyer.com>
Committed: Mon May 21 10:45:53 2018 -0700
----------------------------------------------------------------------
.../stout/include/stout/os/windows/shell.hpp | 135 ++++++++++++++++++-
3rdparty/stout/tests/os_tests.cpp | 53 +++++++-
2 files changed, 176 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/7f115f47/3rdparty/stout/include/stout/os/windows/shell.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/stout/include/stout/os/windows/shell.hpp b/3rdparty/stout/include/stout/os/windows/shell.hpp
index 8da612a..8afcaa6 100644
--- a/3rdparty/stout/include/stout/os/windows/shell.hpp
+++ b/3rdparty/stout/include/stout/os/windows/shell.hpp
@@ -31,6 +31,7 @@
#include <stout/windows.hpp>
#include <stout/os/int_fd.hpp>
+#include <stout/os/pipe.hpp>
#include <stout/internal/windows/inherit.hpp>
@@ -353,8 +354,128 @@ constexpr const char* arg1 = "/c";
} // namespace Shell {
+// Runs a shell command (with `cmd.exe`) with optional arguments.
+//
+// This assumes that a successful execution will result in the exit
+// code for the command to be `0`; in this case, the contents of the
+// `Try` will be the contents of `stdout`.
+//
+// If the exit code is non-zero, we will return an appropriate error
+// message; but *not* `stderr`.
+//
+// If the caller needs to examine the contents of `stderr` it should
+// be redirected to `stdout` (using, e.g., `2>&1 || exit /b 0` in the
+// command string). The `|| exit /b 0` is required to obtain a success
+// exit code in case of errors, and still obtain `stderr`, as piped to
+// `stdout`.
template <typename... T>
-Try<std::string> shell(const std::string& fmt, const T&... t) = delete;
+Try<std::string> shell(const std::string& fmt, const T&... t)
+{
+ using std::array;
+ using std::string;
+ using std::vector;
+
+ const Try<string> command = strings::internal::format(fmt, t...);
+ if (command.isError()) {
+ return Error(command.error());
+ }
+
+ // This function is intended to pass the arguments to the default
+ // shell, so we first add the arguments `cmd.exe cmd.exe /c`,
+ // followed by the command and arguments given.
+ vector<string> args = {os::Shell::name, os::Shell::arg0, os::Shell::arg1};
+
+ { // Minimize the lifetime of the system allocated buffer.
+ //
+ // NOTE: This API returns a pointer to an array of `wchar_t*`,
+ // similar to `argv`. Each pointer to a null-terminated Unicode
+ // string represents an individual argument found on the command
+ // line. We use this because we cannot just split on whitespace.
+ int argc;
+ const std::unique_ptr<wchar_t*, decltype(&::LocalFree)> argv(
+ ::CommandLineToArgvW(wide_stringify(command.get()).data(), &argc),
+ &::LocalFree);
+ if (argv == nullptr) {
+ return WindowsError();
+ }
+
+ for (int i = 0; i < argc; ++i) {
+ args.push_back(stringify(std::wstring(argv.get()[i])));
+ }
+ }
+
+ // This function is intended to return only the `stdout` of the
+ // command; but since we have to redirect all of `stdin`, `stdout`,
+ // `stderr` if we want to redirect any one of them, we redirect
+ // `stdin` and `stderr` to `NUL`, and `stdout` to a pipe.
+ Try<int_fd> stdin_ = os::open(os::DEV_NULL, O_RDONLY);
+ if (stdin_.isError()) {
+ return Error(stdin_.error());
+ }
+
+ Try<array<int_fd, 2>> stdout_ = os::pipe();
+ if (stdout_.isError()) {
+ return Error(stdout_.error());
+ }
+
+ Try<int_fd> stderr_ = os::open(os::DEV_NULL, O_WRONLY);
+ if (stderr_.isError()) {
+ return Error(stderr_.error());
+ }
+
+ // Ensure the file descriptors are closed when we leave this scope.
+ struct Closer
+ {
+ vector<int_fd> fds;
+ ~Closer()
+ {
+ foreach (int_fd& fd, fds) {
+ os::close(fd);
+ }
+ }
+ } closer = {{stdin_.get(), stdout_.get()[0], stderr_.get()}};
+
+ array<int_fd, 3> pipes = {stdin_.get(), stdout_.get()[1], stderr_.get()};
+
+ using namespace ::internal::windows;
+
+ Try<ProcessData> process_data =
+ create_process(args.front(), args, None(), false, pipes);
+
+ if (process_data.isError()) {
+ return Error(process_data.error());
+ }
+
+ // Close the child end of the stdout pipe and then read until EOF.
+ os::close(stdout_.get()[1]);
+ string out;
+ Result<string> part = None();
+ do {
+ part = os::read(stdout_.get()[0], 1024);
+ if (part.isSome()) {
+ out += part.get();
+ }
+ } while (part.isSome());
+
+ // Wait for the process synchronously.
+ ::WaitForSingleObject(process_data->process_handle.get_handle(), INFINITE);
+
+ DWORD status;
+ if (!::GetExitCodeProcess(
+ process_data->process_handle.get_handle(), &status)) {
+ return Error("Failed to `GetExitCodeProcess`: " + command.get());
+ }
+
+ if (status == 0) {
+ return out;
+ }
+
+ return Error(
+ "Failed to execute '" + command.get() +
+ "'; the command was either "
+ "not found or exited with a non-zero exit status: " +
+ stringify(status));
+}
template<typename... T>
@@ -369,8 +490,10 @@ inline Option<int> spawn(
const std::vector<std::string>& arguments,
const Option<std::map<std::string, std::string>>& environment = None())
{
- Try<::internal::windows::ProcessData> process_data =
- ::internal::windows::create_process(command, arguments, environment);
+ using namespace ::internal::windows;
+
+ Try<ProcessData> process_data =
+ create_process(command, arguments, environment);
if (process_data.isError()) {
LOG(WARNING) << process_data.error();
@@ -378,13 +501,11 @@ inline Option<int> spawn(
}
// Wait for the process synchronously.
- ::WaitForSingleObject(
- process_data->process_handle.get_handle(), INFINITE);
+ ::WaitForSingleObject(process_data->process_handle.get_handle(), INFINITE);
DWORD status;
if (!::GetExitCodeProcess(
- process_data->process_handle.get_handle(),
- &status)) {
+ process_data->process_handle.get_handle(), &status)) {
LOG(WARNING) << "Failed to `GetExitCodeProcess`: " << command;
return None();
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/7f115f47/3rdparty/stout/tests/os_tests.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/stout/tests/os_tests.cpp b/3rdparty/stout/tests/os_tests.cpp
index 419879a..b80c34e 100644
--- a/3rdparty/stout/tests/os_tests.cpp
+++ b/3rdparty/stout/tests/os_tests.cpp
@@ -51,6 +51,7 @@
#include <stout/os/kill.hpp>
#include <stout/os/killtree.hpp>
#include <stout/os/realpath.hpp>
+#include <stout/os/shell.hpp>
#include <stout/os/stat.hpp>
#include <stout/os/which.hpp>
#include <stout/os/write.hpp>
@@ -967,34 +968,76 @@ TEST_F(OsTest, Libraries)
#endif // __WINDOWS__
-// NOTE: `os::shell()` is explicitly disallowed on Windows.
-#ifndef __WINDOWS__
TEST_F(OsTest, Shell)
{
Try<string> result = os::shell("echo %s", "hello world");
+#ifdef __WINDOWS__
+ EXPECT_SOME_EQ("hello world\r\n", result);
+#else
EXPECT_SOME_EQ("hello world\n", result);
+#endif // __WINDOWS__
result = os::shell("foobar");
EXPECT_ERROR(result);
+#ifdef __WINDOWS__
+ // NOTE: This relies on the strange semantics of Windows' echo,
+ // where quotes are not removed. We are testing that a quoted
+ // argument with a space in it remains quoted by `os::shell`.
+ result = os::shell("echo \"foo bar\"");
+ EXPECT_SOME_EQ("\"foo bar\"\r\n", result);
+
+ // Because the arguments do not have whitespace in them,
+ // `CommandLineToArgv` removes the surrounding quotes before passing
+ // them to `echo`.
+ result = os::shell("echo \"foo\" \"bar\"");
+ EXPECT_SOME_EQ("foo bar\r\n", result);
+#endif // __WINDOWS__
+
// The `|| true`` necessary so that os::shell() sees a success
// exit code and returns stdout (which we have piped stderr to).
+#ifdef __WINDOWS__
+ result = os::shell("dir foobar889076 2>&1 || exit /b 0");
+ ASSERT_SOME(result);
+ EXPECT_TRUE(strings::contains(result.get(), "File Not Found"));
+#else
result = os::shell("LC_ALL=C ls /tmp/foobar889076 2>&1 || true");
ASSERT_SOME(result);
EXPECT_TRUE(strings::contains(result.get(), "No such file or directory"));
+#endif // __WINDOWS__
+
+ // Testing a command that wrote a substantial amount of data.
+ const string output(2 * os::pagesize(), 'c');
+ const string outfile = path::join(sandbox.get(), "out.txt");
+ ASSERT_SOME(os::write(outfile, output));
+#ifdef __WINDOWS__
+ result = os::shell("type %s", outfile.c_str());
+#else
+ result = os::shell("cat %s", outfile.c_str());
+#endif // __WINDOWS__
+ EXPECT_SOME_EQ(output, result);
// Testing a more ambitious command that mutates the filesystem.
- const string path = "/tmp/os_tests.txt";
+ const string path = path::join(sandbox.get(), "os_tests.txt");
+#ifdef __WINDOWS__
+ result = os::shell("copy /y nul %s", path.c_str());
+ ASSERT_SOME(result);
+ EXPECT_EQ("1 file(s) copied.", strings::trim(result.get()));
+#else
result = os::shell("touch %s", path.c_str());
EXPECT_SOME_EQ("", result);
+#endif // __WINDOWS__
EXPECT_TRUE(os::exists(path));
// Let's clean up, and ensure this worked too.
+#ifdef __WINDOWS__
+ result = os::shell("del %s", path.c_str());
+#else
result = os::shell("rm %s", path.c_str());
+#endif // __WINDOWS__
EXPECT_SOME_EQ("", result);
- EXPECT_FALSE(os::exists("/tmp/os_tests.txt"));
+ EXPECT_FALSE(os::exists(path));
}
-#endif // __WINDOWS__
// NOTE: Disabled on Windows because `mknod` does not exist.