You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by an...@apache.org on 2018/05/21 18:21:53 UTC

[1/2] mesos git commit: Windows: Removed guards now that `os::shell` exists.

Repository: mesos
Updated Branches:
  refs/heads/master 9fa72c549 -> 04263336e


Windows: Removed guards now that `os::shell` exists.

Review: https://reviews.apache.org/r/67189


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/04263336
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/04263336
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/04263336

Branch: refs/heads/master
Commit: 04263336e4cc9880f6733c2f4e113ed608e92532
Parents: 7f115f4
Author: Andrew Schwartzmeyer <an...@schwartzmeyer.com>
Authored: Thu May 17 10:53:30 2018 -0700
Committer: Andrew Schwartzmeyer <an...@schwartzmeyer.com>
Committed: Mon May 21 10:45:53 2018 -0700

----------------------------------------------------------------------
 src/master/main.cpp         |  6 ------
 src/slave/main.cpp          | 12 ------------
 src/tests/fetcher_tests.cpp | 27 +++++----------------------
 3 files changed, 5 insertions(+), 40 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mesos/blob/04263336/src/master/main.cpp
----------------------------------------------------------------------
diff --git a/src/master/main.cpp b/src/master/main.cpp
index 3f5b354..3eb1f89 100644
--- a/src/master/main.cpp
+++ b/src/master/main.cpp
@@ -200,7 +200,6 @@ int main(int argc, char** argv)
   }
 
   if (flags.ip_discovery_command.isSome()) {
-#ifndef __WINDOWS__
     Try<string> ipAddress = os::shell(flags.ip_discovery_command.get());
 
     if (ipAddress.isError()) {
@@ -208,11 +207,6 @@ int main(int argc, char** argv)
     }
 
     os::setenv("LIBPROCESS_IP", strings::trim(ipAddress.get()));
-#else
-    // TODO(andschwa): Support this when `os::shell` is enabled.
-    EXIT(EXIT_FAILURE)
-      << "The `--ip.discovery_command` is not yet supported on Windows";
-#endif // __WINDOWS__
   } else if (flags.ip.isSome()) {
     os::setenv("LIBPROCESS_IP", flags.ip.get());
   }

http://git-wip-us.apache.org/repos/asf/mesos/blob/04263336/src/slave/main.cpp
----------------------------------------------------------------------
diff --git a/src/slave/main.cpp b/src/slave/main.cpp
index ca35668..6461253 100644
--- a/src/slave/main.cpp
+++ b/src/slave/main.cpp
@@ -320,7 +320,6 @@ int main(int argc, char** argv)
   }
 
   if (flags.ip_discovery_command.isSome()) {
-#ifndef __WINDOWS__
     Try<string> ipAddress = os::shell(flags.ip_discovery_command.get());
 
     if (ipAddress.isError()) {
@@ -328,28 +327,17 @@ int main(int argc, char** argv)
     }
 
     os::setenv("LIBPROCESS_IP", strings::trim(ipAddress.get()));
-#else
-    // TODO(andschwa): Support this when `os::shell` is enabled.
-    EXIT(EXIT_FAILURE)
-      << "The `--ip_discovery_command` is not yet supported on Windows";
-#endif // __WINDOWS__
   } else if (flags.ip.isSome()) {
     os::setenv("LIBPROCESS_IP", flags.ip.get());
   }
 
   if (flags.ip6_discovery_command.isSome()) {
-#ifndef __WINDOWS__
     Try<string> ip6Address = os::shell(flags.ip6_discovery_command.get());
     if (ip6Address.isError()) {
       EXIT(EXIT_FAILURE) << ip6Address.error();
     }
 
     os::setenv("LIBPROCESS_IP6", strings::trim(ip6Address.get()));
-#else
-    // TODO(andschwa): Support this when `os::shell` is enabled.
-    EXIT(EXIT_FAILURE)
-      << "The `--ip6_discovery_command` is not yet supported on Windows";
-#endif // __WINDOWS__
   } else if (flags.ip6.isSome()) {
     os::setenv("LIBPROCESS_IP6", flags.ip6.get());
   }

http://git-wip-us.apache.org/repos/asf/mesos/blob/04263336/src/tests/fetcher_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/fetcher_tests.cpp b/src/tests/fetcher_tests.cpp
index 311be6a..95359ec 100644
--- a/src/tests/fetcher_tests.cpp
+++ b/src/tests/fetcher_tests.cpp
@@ -756,10 +756,8 @@ TEST_F_TEMP_DISABLED_ON_WINDOWS(FetcherTest, NoExtractExecutable)
 }
 
 
-// NOTE: This is disabled on Windows because `os::shell()` is deleted.
-// Also, permissions handling needs to be worked out, see MESOS-3176.
-#ifndef __WINDOWS__
-TEST_F(FetcherTest, ExtractNotExecutable)
+// NOTE: Permissions handling needs to be worked out, see MESOS-3176.
+TEST_F_TEMP_DISABLED_ON_WINDOWS(FetcherTest, ExtractNotExecutable)
 {
   // First construct a temporary file that can be fetched and archived with tar
   // gzip.
@@ -811,17 +809,14 @@ TEST_F(FetcherTest, ExtractNotExecutable)
 
   verifyMetrics(1, 0);
 }
-#endif // __WINDOWS__
 
 
 // Tests extracting tar file with extension .tar.
-// NOTE: This is disabled on Windows because `os::shell()` is deleted.
 //
 // Won't be supported on Windows for now; long term thoughts are to perhaps
 // use a code library to provide 'tar' functionality programmatically,
 // see MESOS-8064.
-#ifndef __WINDOWS__
-TEST_F(FetcherTest, ExtractTar)
+TEST_F_TEMP_DISABLED_ON_WINDOWS(FetcherTest, ExtractTar)
 {
   // First construct a temporary file that can be fetched and archived with
   // tar.
@@ -864,16 +859,12 @@ TEST_F(FetcherTest, ExtractTar)
 
   verifyMetrics(1, 0);
 }
-#endif // __WINDOWS__
 
 
-// NOTE: This is disabled on Windows because `os::shell()` is deleted.
-//
 // Won't be supported on Windows for now; long term thoughts are to perhaps
 // use a code library to provide 'gzip' functionality programmatically,
 // see MESOS-8064.
-#ifndef __WINDOWS__
-TEST_F(FetcherTest, ExtractGzipFile)
+TEST_F_TEMP_DISABLED_ON_WINDOWS(FetcherTest, ExtractGzipFile)
 {
   // First construct a temporary file that can be fetched and archived with
   // gzip.
@@ -911,7 +902,6 @@ TEST_F(FetcherTest, ExtractGzipFile)
 
   verifyMetrics(1, 0);
 }
-#endif // __WINDOWS__
 
 
 TEST_F(FetcherTest, UNZIP_ExtractFile)
@@ -1133,8 +1123,6 @@ TEST_F(FetcherTest, UseCustomOutputFile)
 }
 
 
-// NOTE: This is disabled on Windows because `os::shell()` is deleted.
-#ifndef __WINDOWS__
 TEST_F_TEMP_DISABLED_ON_WINDOWS(FetcherTest, CustomGzipOutputFile)
 {
   // First construct a temporary file that can be fetched.
@@ -1174,7 +1162,6 @@ TEST_F_TEMP_DISABLED_ON_WINDOWS(FetcherTest, CustomGzipOutputFile)
 
   verifyMetrics(1, 0);
 }
-#endif // __WINDOWS__
 
 
 // TODO(hausdorff): `os::chmod` does not exist on Windows.
@@ -1275,10 +1262,7 @@ TEST_F(FetcherTest, HdfsURI)
 // agent towards the fetcher. By supplying an invalid SSL setup, we
 // force the fetcher to fail if the parent process does not filter
 // them out.
-//
-// NOTE: This is disabled on Windows because `os::shell()` is deleted.
-#ifndef __WINDOWS__
-TEST_F(FetcherTest, SSLEnvironmentSpillover)
+TEST_F_TEMP_DISABLED_ON_WINDOWS(FetcherTest, SSLEnvironmentSpillover)
 {
   // Patch some critical libprocess environment variables into the
   // parent process of the mesos-fetcher. We expect this test to fail
@@ -1332,7 +1316,6 @@ TEST_F(FetcherTest, SSLEnvironmentSpillover)
     os::setenv("LIBPROCESS_SSL_KEY_FILE", key);
   }
 }
-#endif // __WINDOWS__
 
 } // namespace tests {
 } // namespace internal {


[2/2] mesos git commit: Windows: Implemented `os::shell` and enabled tests.

Posted by an...@apache.org.
Windows: Implemented `os::shell` and enabled tests.

Once implemented, the tests had to be fixed to use Windows commands,
and a test that ensures `os::shell` doesn't hang was added. The
implementation does not reuse `os::spawn` because it must read all the
child process's piped output before waiting on it to exit, otherwise
it will deadlock.

Review: https://reviews.apache.org/r/67188


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/7f115f47
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/7f115f47
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/7f115f47

Branch: refs/heads/master
Commit: 7f115f470e0d9bc8bbb68a2ca16f73b6e318035c
Parents: 9fa72c5
Author: Andrew Schwartzmeyer <an...@schwartzmeyer.com>
Authored: Thu May 17 10:50:05 2018 -0700
Committer: Andrew Schwartzmeyer <an...@schwartzmeyer.com>
Committed: Mon May 21 10:45:53 2018 -0700

----------------------------------------------------------------------
 .../stout/include/stout/os/windows/shell.hpp    | 135 ++++++++++++++++++-
 3rdparty/stout/tests/os_tests.cpp               |  53 +++++++-
 2 files changed, 176 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mesos/blob/7f115f47/3rdparty/stout/include/stout/os/windows/shell.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/stout/include/stout/os/windows/shell.hpp b/3rdparty/stout/include/stout/os/windows/shell.hpp
index 8da612a..8afcaa6 100644
--- a/3rdparty/stout/include/stout/os/windows/shell.hpp
+++ b/3rdparty/stout/include/stout/os/windows/shell.hpp
@@ -31,6 +31,7 @@
 #include <stout/windows.hpp>
 
 #include <stout/os/int_fd.hpp>
+#include <stout/os/pipe.hpp>
 
 #include <stout/internal/windows/inherit.hpp>
 
@@ -353,8 +354,128 @@ constexpr const char* arg1 = "/c";
 
 } // namespace Shell {
 
+// Runs a shell command (with `cmd.exe`) with optional arguments.
+//
+// This assumes that a successful execution will result in the exit
+// code for the command to be `0`; in this case, the contents of the
+// `Try` will be the contents of `stdout`.
+//
+// If the exit code is non-zero, we will return an appropriate error
+// message; but *not* `stderr`.
+//
+// If the caller needs to examine the contents of `stderr` it should
+// be redirected to `stdout` (using, e.g., `2>&1 || exit /b 0` in the
+// command string). The `|| exit /b 0` is required to obtain a success
+// exit code in case of errors, and still obtain `stderr`, as piped to
+// `stdout`.
 template <typename... T>
-Try<std::string> shell(const std::string& fmt, const T&... t) = delete;
+Try<std::string> shell(const std::string& fmt, const T&... t)
+{
+  using std::array;
+  using std::string;
+  using std::vector;
+
+  const Try<string> command = strings::internal::format(fmt, t...);
+  if (command.isError()) {
+    return Error(command.error());
+  }
+
+  // This function is intended to pass the arguments to the default
+  // shell, so we first add the arguments `cmd.exe cmd.exe /c`,
+  // followed by the command and arguments given.
+  vector<string> args = {os::Shell::name, os::Shell::arg0, os::Shell::arg1};
+
+  { // Minimize the lifetime of the system allocated buffer.
+    //
+    // NOTE: This API returns a pointer to an array of `wchar_t*`,
+    // similar to `argv`. Each pointer to a null-terminated Unicode
+    // string represents an individual argument found on the command
+    // line. We use this because we cannot just split on whitespace.
+    int argc;
+    const std::unique_ptr<wchar_t*, decltype(&::LocalFree)> argv(
+      ::CommandLineToArgvW(wide_stringify(command.get()).data(), &argc),
+      &::LocalFree);
+    if (argv == nullptr) {
+      return WindowsError();
+    }
+
+    for (int i = 0; i < argc; ++i) {
+      args.push_back(stringify(std::wstring(argv.get()[i])));
+    }
+  }
+
+  // This function is intended to return only the `stdout` of the
+  // command; but since we have to redirect all of `stdin`, `stdout`,
+  // `stderr` if we want to redirect any one of them, we redirect
+  // `stdin` and `stderr` to `NUL`, and `stdout` to a pipe.
+  Try<int_fd> stdin_ = os::open(os::DEV_NULL, O_RDONLY);
+  if (stdin_.isError()) {
+    return Error(stdin_.error());
+  }
+
+  Try<array<int_fd, 2>> stdout_ = os::pipe();
+  if (stdout_.isError()) {
+    return Error(stdout_.error());
+  }
+
+  Try<int_fd> stderr_ = os::open(os::DEV_NULL, O_WRONLY);
+  if (stderr_.isError()) {
+    return Error(stderr_.error());
+  }
+
+  // Ensure the file descriptors are closed when we leave this scope.
+  struct Closer
+  {
+    vector<int_fd> fds;
+    ~Closer()
+    {
+      foreach (int_fd& fd, fds) {
+        os::close(fd);
+      }
+    }
+  } closer = {{stdin_.get(), stdout_.get()[0], stderr_.get()}};
+
+  array<int_fd, 3> pipes = {stdin_.get(), stdout_.get()[1], stderr_.get()};
+
+  using namespace ::internal::windows;
+
+  Try<ProcessData> process_data =
+    create_process(args.front(), args, None(), false, pipes);
+
+  if (process_data.isError()) {
+    return Error(process_data.error());
+  }
+
+  // Close the child end of the stdout pipe and then read until EOF.
+  os::close(stdout_.get()[1]);
+  string out;
+  Result<string> part = None();
+  do {
+    part = os::read(stdout_.get()[0], 1024);
+    if (part.isSome()) {
+      out += part.get();
+    }
+  } while (part.isSome());
+
+  // Wait for the process synchronously.
+  ::WaitForSingleObject(process_data->process_handle.get_handle(), INFINITE);
+
+  DWORD status;
+  if (!::GetExitCodeProcess(
+        process_data->process_handle.get_handle(), &status)) {
+    return Error("Failed to `GetExitCodeProcess`: " + command.get());
+  }
+
+  if (status == 0) {
+    return out;
+  }
+
+  return Error(
+    "Failed to execute '" + command.get() +
+    "'; the command was either "
+    "not found or exited with a non-zero exit status: " +
+    stringify(status));
+}
 
 
 template<typename... T>
@@ -369,8 +490,10 @@ inline Option<int> spawn(
     const std::vector<std::string>& arguments,
     const Option<std::map<std::string, std::string>>& environment = None())
 {
-  Try<::internal::windows::ProcessData> process_data =
-    ::internal::windows::create_process(command, arguments, environment);
+  using namespace ::internal::windows;
+
+  Try<ProcessData> process_data =
+    create_process(command, arguments, environment);
 
   if (process_data.isError()) {
     LOG(WARNING) << process_data.error();
@@ -378,13 +501,11 @@ inline Option<int> spawn(
   }
 
   // Wait for the process synchronously.
-  ::WaitForSingleObject(
-      process_data->process_handle.get_handle(), INFINITE);
+  ::WaitForSingleObject(process_data->process_handle.get_handle(), INFINITE);
 
   DWORD status;
   if (!::GetExitCodeProcess(
-           process_data->process_handle.get_handle(),
-           &status)) {
+        process_data->process_handle.get_handle(), &status)) {
     LOG(WARNING) << "Failed to `GetExitCodeProcess`: " << command;
     return None();
   }

http://git-wip-us.apache.org/repos/asf/mesos/blob/7f115f47/3rdparty/stout/tests/os_tests.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/stout/tests/os_tests.cpp b/3rdparty/stout/tests/os_tests.cpp
index 419879a..b80c34e 100644
--- a/3rdparty/stout/tests/os_tests.cpp
+++ b/3rdparty/stout/tests/os_tests.cpp
@@ -51,6 +51,7 @@
 #include <stout/os/kill.hpp>
 #include <stout/os/killtree.hpp>
 #include <stout/os/realpath.hpp>
+#include <stout/os/shell.hpp>
 #include <stout/os/stat.hpp>
 #include <stout/os/which.hpp>
 #include <stout/os/write.hpp>
@@ -967,34 +968,76 @@ TEST_F(OsTest, Libraries)
 #endif // __WINDOWS__
 
 
-// NOTE: `os::shell()` is explicitly disallowed on Windows.
-#ifndef __WINDOWS__
 TEST_F(OsTest, Shell)
 {
   Try<string> result = os::shell("echo %s", "hello world");
+#ifdef __WINDOWS__
+  EXPECT_SOME_EQ("hello world\r\n", result);
+#else
   EXPECT_SOME_EQ("hello world\n", result);
+#endif // __WINDOWS__
 
   result = os::shell("foobar");
   EXPECT_ERROR(result);
 
+#ifdef __WINDOWS__
+  // NOTE: This relies on the strange semantics of Windows' echo,
+  // where quotes are not removed. We are testing that a quoted
+  // argument with a space in it remains quoted by `os::shell`.
+  result = os::shell("echo \"foo bar\"");
+  EXPECT_SOME_EQ("\"foo bar\"\r\n", result);
+
+  // Because the arguments do not have whitespace in them,
+  // `CommandLineToArgv` removes the surrounding quotes before passing
+  // them to `echo`.
+  result = os::shell("echo \"foo\" \"bar\"");
+  EXPECT_SOME_EQ("foo bar\r\n", result);
+#endif // __WINDOWS__
+
   // The `|| true`` necessary so that os::shell() sees a success
   // exit code and returns stdout (which we have piped stderr to).
+#ifdef __WINDOWS__
+  result = os::shell("dir foobar889076 2>&1 || exit /b 0");
+  ASSERT_SOME(result);
+  EXPECT_TRUE(strings::contains(result.get(), "File Not Found"));
+#else
   result = os::shell("LC_ALL=C ls /tmp/foobar889076 2>&1 || true");
   ASSERT_SOME(result);
   EXPECT_TRUE(strings::contains(result.get(), "No such file or directory"));
+#endif // __WINDOWS__
+
+  // Testing a command that wrote a substantial amount of data.
+  const string output(2 * os::pagesize(), 'c');
+  const string outfile = path::join(sandbox.get(), "out.txt");
+  ASSERT_SOME(os::write(outfile, output));
+#ifdef __WINDOWS__
+  result = os::shell("type %s", outfile.c_str());
+#else
+  result = os::shell("cat %s", outfile.c_str());
+#endif // __WINDOWS__
+  EXPECT_SOME_EQ(output, result);
 
   // Testing a more ambitious command that mutates the filesystem.
-  const string path = "/tmp/os_tests.txt";
+  const string path = path::join(sandbox.get(), "os_tests.txt");
+#ifdef __WINDOWS__
+  result = os::shell("copy /y nul %s", path.c_str());
+  ASSERT_SOME(result);
+  EXPECT_EQ("1 file(s) copied.", strings::trim(result.get()));
+#else
   result = os::shell("touch %s", path.c_str());
   EXPECT_SOME_EQ("", result);
+#endif // __WINDOWS__
   EXPECT_TRUE(os::exists(path));
 
   // Let's clean up, and ensure this worked too.
+#ifdef __WINDOWS__
+  result = os::shell("del %s", path.c_str());
+#else
   result = os::shell("rm %s", path.c_str());
+#endif // __WINDOWS__
   EXPECT_SOME_EQ("", result);
-  EXPECT_FALSE(os::exists("/tmp/os_tests.txt"));
+  EXPECT_FALSE(os::exists(path));
 }
-#endif // __WINDOWS__
 
 
 // NOTE: Disabled on Windows because `mknod` does not exist.