You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by jo...@apache.org on 2016/08/27 01:20:00 UTC
[1/6] mesos git commit: Used `os::execvp` instead of `::execvp` in
the Mesos launcher.
Repository: mesos
Updated Branches:
refs/heads/master bda9a5087 -> 537584cf2
Used `os::execvp` instead of `::execvp` in the Mesos launcher.
Review: https://reviews.apache.org/r/51213/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/ec30c1f2
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/ec30c1f2
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/ec30c1f2
Branch: refs/heads/master
Commit: ec30c1f22466362e5dd1205d473fc652ad97b644
Parents: 8509553
Author: Daniel Pravat <dp...@outlook.com>
Authored: Fri Aug 26 17:23:21 2016 -0700
Committer: Joseph Wu <jo...@apache.org>
Committed: Fri Aug 26 18:19:02 2016 -0700
----------------------------------------------------------------------
src/slave/containerizer/mesos/launch.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/ec30c1f2/src/slave/containerizer/mesos/launch.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/launch.cpp b/src/slave/containerizer/mesos/launch.cpp
index 2db8db5..13b65d8 100644
--- a/src/slave/containerizer/mesos/launch.cpp
+++ b/src/slave/containerizer/mesos/launch.cpp
@@ -392,7 +392,7 @@ int MesosContainerizerLaunch::execute()
(char*) nullptr);
} else {
// Use execvp to launch the command.
- execvp(command->value().c_str(),
+ os::execvp(command->value().c_str(),
os::raw::Argv(command->arguments()));
}
[3/6] mesos git commit: Added `os::execvp' in Stout.
Posted by jo...@apache.org.
Added `os::execvp' in Stout.
On Windows, the semantics of `execvp` are emulated via spawning
and waiting on a child process.
Review: https://reviews.apache.org/r/51217/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/8509553d
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/8509553d
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/8509553d
Branch: refs/heads/master
Commit: 8509553def821536077a96bdee1ac2a8a5509c7a
Parents: a5f6df6
Author: Daniel Pravat <dp...@outlook.com>
Authored: Fri Aug 26 17:23:20 2016 -0700
Committer: Joseph Wu <jo...@apache.org>
Committed: Fri Aug 26 18:19:02 2016 -0700
----------------------------------------------------------------------
3rdparty/stout/include/stout/os/posix/shell.hpp | 3 +++
3rdparty/stout/include/stout/os/windows/shell.hpp | 14 ++++++++++++++
2 files changed, 17 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/8509553d/3rdparty/stout/include/stout/os/posix/shell.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/stout/include/stout/os/posix/shell.hpp b/3rdparty/stout/include/stout/os/posix/shell.hpp
index 1d73ae5..df8fb4c 100644
--- a/3rdparty/stout/include/stout/os/posix/shell.hpp
+++ b/3rdparty/stout/include/stout/os/posix/shell.hpp
@@ -30,6 +30,9 @@
namespace os {
+// Import `::execvp` into `os::` namespace.
+using ::execvp;
+
namespace Shell {
// Canonical constants used as platform-dependent args to `exec`
http://git-wip-us.apache.org/repos/asf/mesos/blob/8509553d/3rdparty/stout/include/stout/os/windows/shell.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/stout/include/stout/os/windows/shell.hpp b/3rdparty/stout/include/stout/os/windows/shell.hpp
index c6e141a..8fb48c1 100644
--- a/3rdparty/stout/include/stout/os/windows/shell.hpp
+++ b/3rdparty/stout/include/stout/os/windows/shell.hpp
@@ -104,6 +104,10 @@ inline int system(const std::string& command)
}
+// On Windows, the `_spawnlp` call creates a new process.
+// In order to emulate the semantics of `execlp`, we spawn with `_P_WAIT`,
+// which forces the parent process to block on the child. When the child exits,
+// the exit code is propagated back through the parent via `exit()`.
template<typename... T>
inline int execlp(const char* file, T... t)
{
@@ -111,6 +115,16 @@ inline int execlp(const char* file, T... t)
}
+// On Windows, the `_spawnvp` call creates a new process.
+// In order to emulate the semantics of `execvp`, we spawn with `_P_WAIT`,
+// which forces the parent process to block on the child. When the child exits,
+// the exit code is propagated back through the parent via `exit()`.
+inline int execvp(const char* file, char* const argv[])
+{
+ exit(::_spawnvp(_P_WAIT, file, argv));
+}
+
+
// Concatenates multiple command-line arguments and escapes the values.
// If `arg` is not specified (or takes the value `0`), the function will
// scan `argv` until a `nullptr` is encountered.
[6/6] mesos git commit: Windows: Added `userenv` import library.
Posted by jo...@apache.org.
Windows: Added `userenv` import library.
This is used to check the system environment.
Review: https://reviews.apache.org/r/51234/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/537584cf
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/537584cf
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/537584cf
Branch: refs/heads/master
Commit: 537584cf211ac193c20be63dfcb5b48953817e1e
Parents: 87ba0ef
Author: Daniel Pravat <dp...@outlook.com>
Authored: Fri Aug 26 17:23:35 2016 -0700
Committer: Joseph Wu <jo...@apache.org>
Committed: Fri Aug 26 18:19:03 2016 -0700
----------------------------------------------------------------------
3rdparty/libprocess/cmake/ProcessConfigure.cmake | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/537584cf/3rdparty/libprocess/cmake/ProcessConfigure.cmake
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/cmake/ProcessConfigure.cmake b/3rdparty/libprocess/cmake/ProcessConfigure.cmake
index afdb1fe..873f41d 100755
--- a/3rdparty/libprocess/cmake/ProcessConfigure.cmake
+++ b/3rdparty/libprocess/cmake/ProcessConfigure.cmake
@@ -127,6 +127,13 @@ set(PROCESS_LIBS
${HTTP_PARSER_LFLAG}
)
+if (WIN32)
+ set(PROCESS_LIBS
+ ${PROCESS_LIBS}
+ userenv
+ )
+endif (WIN32)
+
if (NOT ENABLE_LIBEVENT)
set(PROCESS_LIBS ${PROCESS_LIBS} ${LIBEV_LFLAG})
elseif (ENABLE_LIBEVENT)
[4/6] mesos git commit: Windows: Corrected subprocess environment
variables.
Posted by jo...@apache.org.
Windows: Corrected subprocess environment variables.
Windows processes have a minimum set of environment variables required
to load prerequisites, such as shared libraries. These environment
variables are generally found in the system's environment.
Review: https://reviews.apache.org/r/51233/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/9d378635
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/9d378635
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/9d378635
Branch: refs/heads/master
Commit: 9d3786353b4a4c357f9196d9659bac4582dd0176
Parents: ec30c1f
Author: Daniel Pravat <dp...@outlook.com>
Authored: Fri Aug 26 17:23:22 2016 -0700
Committer: Joseph Wu <jo...@apache.org>
Committed: Fri Aug 26 18:19:03 2016 -0700
----------------------------------------------------------------------
.../include/process/windows/subprocess.hpp | 70 +++++++++++++++++++-
1 file changed, 69 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/9d378635/3rdparty/libprocess/include/process/windows/subprocess.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/include/process/windows/subprocess.hpp b/3rdparty/libprocess/include/process/windows/subprocess.hpp
index 6bb54c8..f452f67 100644
--- a/3rdparty/libprocess/include/process/windows/subprocess.hpp
+++ b/3rdparty/libprocess/include/process/windows/subprocess.hpp
@@ -30,6 +30,8 @@
#include <stout/os/close.hpp>
#include <stout/os/environment.hpp>
+#include <userEnv.h>
+
using std::map;
using std::string;
using std::vector;
@@ -62,6 +64,56 @@ inline void close(
}
}
+// Retrieves system environment in a `std::map`, ignoring
+// the current process's environment variables.
+inline Option<map<string, string>> getSystemEnvironment()
+{
+ std::wstring_convert<std::codecvt<wchar_t, char, mbstate_t>,
+ wchar_t> converter;
+
+ map<string, string> systemEnvironment;
+ wchar_t* environmentEntry = nullptr;
+
+ // Get the system environment.
+ // The third parameter (bool) tells the function *not* to inherit
+ // variables from the current process.
+ if (!CreateEnvironmentBlock((LPVOID*)&environmentEntry, nullptr, FALSE)) {
+ return None();
+ }
+
+ // Save the environment block in order to destroy it later.
+ wchar_t* environmentBlock = environmentEntry;
+
+ while (*environmentEntry != L'\0') {
+ // Each environment block contains the environment variables as follows:
+ // Var1=Value1\0
+ // Var2=Value2\0
+ // Var3=Value3\0
+ // ...
+ // VarN=ValueN\0\0
+ // The name of an environment variable cannot include an equal sign (=).
+
+ wchar_t * separator = wcschr(environmentEntry, L'=');
+ std::wstring varName = std::wstring(environmentEntry, separator);
+ std::wstring varVal = std::wstring(separator + 1);
+
+ // Mesos variables are upper case. Convert system variables to
+ // match the name provided by the scheduler in case of a collision.
+ std::transform(varName.begin(), varName.end(), varName.begin(), ::towupper);
+
+ // The system environment has priority. Force `ANSI` usage until the code
+ // is converted to UNICODE.
+ systemEnvironment.insert_or_assign(
+ converter.to_bytes(varName.c_str()),
+ converter.to_bytes(varVal.c_str()));
+
+ environmentEntry += varName.length() + varVal.length() + 2;
+ }
+
+ DestroyEnvironmentBlock(environmentBlock);
+
+ return systemEnvironment;
+}
// Creates a null-terminated array of null-terminated strings that will be
// passed to `CreateProcess` as the `lpEnvironment` argument, as described by
@@ -70,6 +122,10 @@ inline void close(
// environments, so it should not be used in conjunction with the
// `CREATE_UNICODE_ENVIRONMENT` flag.
//
+// NOTE: This function will add the system's environment variables into
+// the returned string. These variables take precedence over the provided
+// `env` and are generally necessary in order to launch things on Windows.
+//
// [1] https://msdn.microsoft.com/en-us/library/windows/desktop/ms682425(v=vs.85).aspx
inline Option<string> createProcessEnvironment(
const Option<map<string, string>>& env)
@@ -78,8 +134,20 @@ inline Option<string> createProcessEnvironment(
return None();
}
+ Option<map<string, string>> systemEnvironment = getSystemEnvironment();
+
+ // The system environment must be non-empty.
+ // No subprocesses will be able to launch if the system environment is blank.
+ CHECK(systemEnvironment.isSome() && systemEnvironment.get().size() > 0);
+
+ map<string, string> combinedEnvironment = env.get();
+
+ foreachpair(const string& key, const string& value, systemEnvironment.get()) {
+ combinedEnvironment[key] = value;
+ }
+
string environmentString;
- foreachpair (const string& key, const string& value, env.get()) {
+ foreachpair (const string& key, const string& value, combinedEnvironment) {
environmentString += key + '=' + value + '\0';
}
[2/6] mesos git commit: Windows: Update mesos-executor name.
Posted by jo...@apache.org.
Windows: Update mesos-executor name.
This replaces the `mesos-executor` string with `mesos-executor.exe`
on Windows to disambiguate between potential extensions.
Review: https://reviews.apache.org/r/51210/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/a5f6df60
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/a5f6df60
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/a5f6df60
Branch: refs/heads/master
Commit: a5f6df606ea6b51bf3eb979d200377ff0959934e
Parents: bda9a50
Author: Daniel Pravat <dp...@outlook.com>
Authored: Fri Aug 26 17:23:18 2016 -0700
Committer: Joseph Wu <jo...@apache.org>
Committed: Fri Aug 26 18:19:02 2016 -0700
----------------------------------------------------------------------
src/slave/slave.cpp | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/a5f6df60/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index 5d162d0..d088d7c 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -125,6 +125,12 @@ using process::Owned;
using process::Time;
using process::UPID;
+#ifdef __WINDOWS__
+constexpr char MESOS_EXECUTOR[] = "mesos-executor.exe";
+#else
+constexpr char MESOS_EXECUTOR[] = "mesos-executor";
+#endif // __WINDOWS__
+
namespace mesos {
namespace internal {
namespace slave {
@@ -3956,12 +3962,12 @@ ExecutorInfo Slave::getExecutorInfo(
}
Result<string> path = os::realpath(
- path::join(flags.launcher_dir, "mesos-executor"));
+ path::join(flags.launcher_dir, MESOS_EXECUTOR));
if (path.isSome()) {
executor.mutable_command()->set_shell(false);
executor.mutable_command()->set_value(path.get());
- executor.mutable_command()->add_arguments("mesos-executor");
+ executor.mutable_command()->add_arguments(MESOS_EXECUTOR);
executor.mutable_command()->add_arguments(
"--launcher_dir=" + flags.launcher_dir);
@@ -6072,7 +6078,7 @@ Executor::Executor(
CHECK_NOTNULL(slave);
Result<string> executorPath =
- os::realpath(path::join(slave->flags.launcher_dir, "mesos-executor"));
+ os::realpath(path::join(slave->flags.launcher_dir, MESOS_EXECUTOR));
if (executorPath.isSome()) {
commandExecutor =
[5/6] mesos git commit: Windows: Excluded UNIX-style `PATH`
environment variable.
Posted by jo...@apache.org.
Windows: Excluded UNIX-style `PATH` environment variable.
The default `PATH` used to launch an executor is UNIX-specific.
On Windows, this `PATH` is determined by system settings.
Review: https://reviews.apache.org/r/51462/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/87ba0efa
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/87ba0efa
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/87ba0efa
Branch: refs/heads/master
Commit: 87ba0efa889d744a92f7d6562dbaf5c6c8df6167
Parents: 9d37863
Author: Daniel Pravat <dp...@outlook.com>
Authored: Fri Aug 26 17:23:23 2016 -0700
Committer: Joseph Wu <jo...@apache.org>
Committed: Fri Aug 26 18:19:03 2016 -0700
----------------------------------------------------------------------
src/slave/slave.cpp | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/87ba0efa/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index d088d7c..1166477 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -6340,11 +6340,15 @@ map<string, string> executorEnvironment(
}
}
+#ifndef __WINDOWS__
// Include a default $PATH if there isn't.
+ // On Windows, we ensure the path is set by checking the system environment.
+ // See `createProcessEnvironment` in `process/windows/subprocess.hpp`.
if (environment.count("PATH") == 0) {
environment["PATH"] =
"/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
}
+#endif // __WINDOWS__
// Set LIBPROCESS_PORT so that we bind to a random free port (since
// this might have been set via --port option). We do this before