You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by jo...@apache.org on 2016/06/07 20:21:33 UTC
[1/2] mesos git commit: Remove SASL dependency for Windows builds.
Repository: mesos
Updated Branches:
refs/heads/master 711210d42 -> 0e331b0db
Remove SASL dependency for Windows builds.
SASL is currently a hard dependency for Mesos. Per MESOS-5450, we expect
that some platforms will not support SASL (namely Windows), so this
commit will begin the first step in a several-step process of removing
it as a hard dependency.
For this first step, this commit will shed SASL dependency in libmesos
only for Windows builds. We do this by:
(1) Adding a preprocessor symbol, `HAS_AUTHENTICATION` that wraps the
SASL-dependent code, so that we can conditionally choose not to
compile it.
(2) Defining `HAS_AUTHENTICATION` on all Unix builds, and leaving it
undefined on all Windows builds.
(3) Logging an error and exiting the master if the user passes in
flags that depend on SASL, such as `--authenticate`.
Notably, what we do *not* do is:
(1) Shed SASL dependency in the tests.
The impact of this is that, on Windows, relevant libmesos tests will
either not compile, or not pass. Naturally, as tracked by MESOS-5450,
the second phase of this series will be to shed the test dependency as
well.
Review: https://reviews.apache.org/r/47821/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/99145d3e
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/99145d3e
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/99145d3e
Branch: refs/heads/master
Commit: 99145d3e996e08eb7f77a55b1ba030f071fb0f08
Parents: 711210d
Author: Alex Clemmer <cl...@gmail.com>
Authored: Tue Jun 7 15:59:46 2016 -0400
Committer: Joris Van Remoortere <jo...@gmail.com>
Committed: Tue Jun 7 16:20:58 2016 -0400
----------------------------------------------------------------------
cmake/CompilationConfigure.cmake | 10 ++++++++++
src/Makefile.am | 9 +++++++++
src/master/master.cpp | 18 ++++++++++++++++++
src/sched/sched.cpp | 17 ++++++++++++++++-
4 files changed, 53 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/99145d3e/cmake/CompilationConfigure.cmake
----------------------------------------------------------------------
diff --git a/cmake/CompilationConfigure.cmake b/cmake/CompilationConfigure.cmake
index d3a5dae..9325e5f 100644
--- a/cmake/CompilationConfigure.cmake
+++ b/cmake/CompilationConfigure.cmake
@@ -128,6 +128,16 @@ if (WIN32)
# [2] https://code.google.com/p/google-glog/source/browse/trunk/src/windows/glog/logging.h?r=113
add_definitions(-DNOGDI)
add_definitions(-DNOMINMAX)
+else (WIN32)
+ # TODO(hausdorff): Remove our hard dependency on SASL, as some platforms
+ # (namely Windows) will not support it in the forseeable future. As a
+ # stop-gap, we conditionally compile the code in libmesos that depends on
+ # SASL, by always setting `HAS_AUTHENTICATION` on non-Windows platforms, and
+ # never setting it on Windows platforms. This means that non-Windows builds
+ # of libmesos will still take a hard dependency on SASL, while Windows builds
+ # won't. Currently, the dependency is still assumed throughout the tests,
+ # though the plan is to remove this hard dependency as well. See MESOS-5450.
+ add_definitions(-DHAS_AUTHENTICATION=1)
endif (WIN32)
# Enable the INT64 support for PicoJSON.
http://git-wip-us.apache.org/repos/asf/mesos/blob/99145d3e/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index b02b901..29525c9 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -187,6 +187,15 @@ LIB_NVIDIA = -lnvidia-ml
LDADD += -lnvidia-ml
endif
+# TODO(hausdorff): Remove our hard dependency on SASL, as some platforms
+# (namely Windows) will not support it in the forseeable future. As a
+# stop-gap, we conditionally compile the code in libmesos that depends on
+# SASL, by always setting `HAS_AUTHENTICATION` on non-Windows platforms, and
+# never setting it on Windows platforms. This means that non-Windows builds
+# of libmesos will still take a hard dependency on SASL, while Windows builds
+# won't. Currently, the dependency is still assumed throughout the tests,
+# though the plan is to remove this hard dependency as well. See MESOS-5450.
+MESOS_CPPFLAGS += -DHAS_AUTHENTICATION=1
# README: we build the Mesos library out of a collection of
# convenience libraries (that is, libraries that do not get installed
http://git-wip-us.apache.org/repos/asf/mesos/blob/99145d3e/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index ecefeb3..21ec704 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -428,18 +428,34 @@ void Master::initialize()
<< " Must be within [0%-100%]";
}
+#ifdef HAS_AUTHENTICATION
// Log authentication state.
if (flags.authenticate_frameworks) {
LOG(INFO) << "Master only allowing authenticated frameworks to register";
} else {
LOG(INFO) << "Master allowing unauthenticated frameworks to register";
}
+#else
+ if (flags.authenticate_frameworks) {
+ EXIT(EXIT_FAILURE) << "Authentication is not supported on this platform, "
+ "but --authenticate flag was passed as argument to "
+ "master";
+ }
+#endif // HAS_AUTHENTICATION
+#ifdef HAS_AUTHENTICATION
if (flags.authenticate_agents) {
LOG(INFO) << "Master only allowing authenticated agents to register";
} else {
LOG(INFO) << "Master allowing unauthenticated agents to register";
}
+#else
+ if (flags.authenticate_agents) {
+ EXIT(EXIT_FAILURE) << "Authentication is not supported on this platform, "
+ "but --authenticate_slaves was passed as argument to "
+ "master";
+ }
+#endif // HAS_AUTHENTICATION
if (flags.authenticate_http_frameworks) {
LOG(INFO) << "Master only allowing authenticated HTTP frameworks to "
@@ -465,6 +481,7 @@ void Master::initialize()
credentials = _credentials.get();
}
+#ifdef HAS_AUTHENTICATION
// Extract authenticator names and validate them.
authenticatorNames = strings::split(flags.authenticators, ",");
if (authenticatorNames.empty()) {
@@ -522,6 +539,7 @@ void Master::initialize()
authenticator = None();
}
}
+#endif // HAS_AUTHENTICATION
// TODO(arojas): Consider creating a factory function for the instantiation
// of the HTTP authenticator the same way the allocator does.
http://git-wip-us.apache.org/repos/asf/mesos/blob/99145d3e/src/sched/sched.cpp
----------------------------------------------------------------------
diff --git a/src/sched/sched.cpp b/src/sched/sched.cpp
index 80c5cfc..9f561d7 100644
--- a/src/sched/sched.cpp
+++ b/src/sched/sched.cpp
@@ -328,6 +328,7 @@ protected:
LOG(INFO) << "New master detected at " << master.get().pid();
link(master.get().pid());
+#ifdef HAS_AUTHENTICATION
if (credential.isSome()) {
// Authenticate with the master.
// TODO(vinod): Do a backoff for authentication similar to what
@@ -344,6 +345,15 @@ protected:
// (e.g., rate limiting tests).
doReliableRegistration(flags.registration_backoff_factor);
}
+#else
+ // Authentication not enabled on this platform. Proceed with registration
+ // without authentication.
+ reauthenticate = false;
+ LOG(INFO) << "Authentication is not available on this platform. "
+ "Attempting to register without authentication";
+
+ doReliableRegistration(flags.registration_backoff_factor);
+#endif // HAS_AUTHENTICATION
} else {
// In this case, we don't actually invoke Scheduler::error
// since we might get reconnected to a master imminently.
@@ -355,7 +365,7 @@ protected:
.onAny(defer(self(), &SchedulerProcess::detected, lambda::_1));
}
-
+#ifdef HAS_AUTHENTICATION
void authenticate()
{
if (!running.load()) {
@@ -496,6 +506,7 @@ protected:
LOG(WARNING) << "Authentication timed out";
}
}
+#endif // HAS_AUTHENTICATION
void drop(const Event& event, const string& message)
{
@@ -778,9 +789,13 @@ protected:
return;
}
+#ifdef HAS_AUTHENTICATION
if (credential.isSome() && !authenticated) {
return;
}
+#else
+ authenticated = false;
+#endif // HAS_AUTHENTICATION
VLOG(1) << "Sending SUBSCRIBE call to " << master.get().pid();
[2/2] mesos git commit: Agent: Added minor changes to various .cpp
files to support Windows.
Posted by jo...@apache.org.
Agent: Added minor changes to various .cpp files to support Windows.
Review: https://reviews.apache.org/r/47577/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/0e331b0d
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/0e331b0d
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/0e331b0d
Branch: refs/heads/master
Commit: 0e331b0db19211526e043ed8c6ddfc1725821259
Parents: 99145d3
Author: Alex Clemmer <cl...@gmail.com>
Authored: Tue Jun 7 15:54:55 2016 -0400
Committer: Joris Van Remoortere <jo...@gmail.com>
Committed: Tue Jun 7 16:21:14 2016 -0400
----------------------------------------------------------------------
src/docker/docker.cpp | 9 +++++----
src/docker/executor.cpp | 2 ++
src/files/files.cpp | 9 ++++++---
src/health-check/main.cpp | 8 ++++++--
src/logging/logging.cpp | 10 +++++++++-
src/slave/slave.cpp | 7 ++++++-
src/uri/fetcher.cpp | 3 +++
src/uri/fetcher.hpp | 5 +++++
src/uri/fetchers/copy.cpp | 2 ++
src/uri/fetchers/curl.cpp | 2 ++
src/zookeeper/group.cpp | 12 ++++++++++--
11 files changed, 56 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/0e331b0d/src/docker/docker.cpp
----------------------------------------------------------------------
diff --git a/src/docker/docker.cpp b/src/docker/docker.cpp
index 19cf424..ffbd7ef 100755
--- a/src/docker/docker.cpp
+++ b/src/docker/docker.cpp
@@ -22,6 +22,7 @@
#include <stout/result.hpp>
#include <stout/strings.hpp>
+#include <stout/os/killtree.hpp>
#include <stout/os/read.hpp>
#include <process/check.hpp>
@@ -447,8 +448,8 @@ Future<Nothing> Docker::run(
const string& mappedDirectory,
const Option<Resources>& resources,
const Option<map<string, string>>& env,
- const process::Subprocess::IO& stdout,
- const process::Subprocess::IO& stderr) const
+ const process::Subprocess::IO& _stdout,
+ const process::Subprocess::IO& _stderr) const
{
if (!containerInfo.has_docker()) {
return Failure("No docker info found in container info");
@@ -686,8 +687,8 @@ Future<Nothing> Docker::run(
path,
argv,
Subprocess::PATH("/dev/null"),
- stdout,
- stderr,
+ _stdout,
+ _stderr,
NO_SETSID,
None(),
environment);
http://git-wip-us.apache.org/repos/asf/mesos/blob/0e331b0d/src/docker/executor.cpp
----------------------------------------------------------------------
diff --git a/src/docker/executor.cpp b/src/docker/executor.cpp
index f4796e4..1b3a779 100644
--- a/src/docker/executor.cpp
+++ b/src/docker/executor.cpp
@@ -35,6 +35,8 @@
#include <stout/protobuf.hpp>
#include <stout/try.hpp>
+#include <stout/os/killtree.hpp>
+
#include "common/status_utils.hpp"
#include "docker/docker.hpp"
http://git-wip-us.apache.org/repos/asf/mesos/blob/0e331b0d/src/files/files.cpp
----------------------------------------------------------------------
diff --git a/src/files/files.cpp b/src/files/files.cpp
index e104c6d..55c310a 100644
--- a/src/files/files.cpp
+++ b/src/files/files.cpp
@@ -50,6 +50,8 @@
#include <stout/strings.hpp>
#include <stout/try.hpp>
+#include <stout/os/constants.hpp>
+
#include "files/files.hpp"
#include "logging/logging.hpp"
@@ -590,7 +592,7 @@ Future<Response> FilesProcess::_read(
offset,
data,
jsonp))
- .onAny(lambda::bind(&os::close, fd.get()));
+ .onAny([fd]() { os::close(fd.get()); });
}
@@ -708,7 +710,8 @@ Result<string> FilesProcess::resolve(const string& path)
// longest possible prefix match and if found append any suffix to
// the attached path (provided the path is to a directory).
vector<string> tokens = strings::split(
- strings::remove(path, "/", strings::SUFFIX), "/");
+ strings::remove(path, stringify(os::PATH_SEPARATOR), strings::SUFFIX),
+ stringify(os::PATH_SEPARATOR));
string suffix;
while (!tokens.empty()) {
@@ -730,7 +733,7 @@ Result<string> FilesProcess::resolve(const string& path)
// 'Not Found'.
string path = paths[prefix];
if (os::stat::isdir(path)) {
- path = path::join(path, suffix);
+ path = path::join(path, suffix, os::PATH_SEPARATOR);
// Canonicalize the absolute path.
Result<string> realpath = os::realpath(path);
http://git-wip-us.apache.org/repos/asf/mesos/blob/0e331b0d/src/health-check/main.cpp
----------------------------------------------------------------------
diff --git a/src/health-check/main.cpp b/src/health-check/main.cpp
index 3cf8583..4cc9dde 100644
--- a/src/health-check/main.cpp
+++ b/src/health-check/main.cpp
@@ -17,7 +17,9 @@
#include <signal.h>
#include <stdio.h>
#include <string.h>
+#ifndef __WINDOWS__
#include <unistd.h>
+#endif // __WINDOWS__
#include <iostream>
#include <string>
@@ -43,6 +45,8 @@
#include <stout/protobuf.hpp>
#include <stout/strings.hpp>
+#include <stout/os/killtree.hpp>
+
#include "common/status_utils.hpp"
#include "messages/messages.hpp"
@@ -374,7 +378,7 @@ int main(int argc, char** argv)
TaskID taskID;
taskID.set_value(flags.task_id.get());
- internal::HealthCheckerProcess process(
+ mesos::internal::HealthCheckerProcess process(
check.get(),
flags.executor.get(),
taskID);
@@ -383,7 +387,7 @@ int main(int argc, char** argv)
process::Future<Nothing> checking =
process::dispatch(
- process, &internal::HealthCheckerProcess::healthCheck);
+ process, &mesos::internal::HealthCheckerProcess::healthCheck);
checking.await();
http://git-wip-us.apache.org/repos/asf/mesos/blob/0e331b0d/src/logging/logging.cpp
----------------------------------------------------------------------
diff --git a/src/logging/logging.cpp b/src/logging/logging.cpp
index 7f2d843..c7ac957 100644
--- a/src/logging/logging.cpp
+++ b/src/logging/logging.cpp
@@ -103,6 +103,7 @@ inline void handler(int signal, siginfo_t *siginfo, void *context)
}
#endif // __WINDOWS__
+
google::LogSeverity getLogSeverity(const string& logging_level)
{
if (logging_level == "INFO") {
@@ -199,7 +200,7 @@ void initialize(
(flags.log_dir.isSome() ? flags.log_dir.get() : "STDERR");
if (installFailureSignalHandler) {
- // glog on Windows does not support `InstallFailureSignalHandler`.
+ // glog on Windows does not support `InstallFailureSignalHandler`.
#ifndef __WINDOWS__
// Handles SIGSEGV, SIGILL, SIGFPE, SIGABRT, SIGBUS, SIGTERM
// by default.
@@ -212,6 +213,13 @@ void initialize(
// [1] https://msdn.microsoft.com/en-us/library/xdkz3x12.aspx
// Set up our custom signal handlers.
+ //
+ // NOTE: The code below sets the SIGTERM signal handler to the `handle`
+ // function declared above. While this is useful on POSIX systems, SIGTERM
+ // is generated and handled differently on Windows[1], so this code would
+ // not work.
+ //
+ // [1] https://msdn.microsoft.com/en-us/library/xdkz3x12.aspx
struct sigaction action;
action.sa_sigaction = handler;
http://git-wip-us.apache.org/repos/asf/mesos/blob/0e331b0d/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index 03a845e..d635dd2 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -1029,10 +1029,15 @@ void Slave::authenticate()
CHECK(authenticatee == nullptr);
+#ifdef HAS_AUTHENTICATION
+ // On Windows CRAMMD5Authenticatee is not supported.
if (authenticateeName == DEFAULT_AUTHENTICATEE) {
LOG(INFO) << "Using default CRAM-MD5 authenticatee";
authenticatee = new cram_md5::CRAMMD5Authenticatee();
- } else {
+ }
+#endif // HAS_AUTHENTICATION
+
+ if (authenticatee == NULL) {
Try<Authenticatee*> module =
modules::ModuleManager::create<Authenticatee>(authenticateeName);
if (module.isError()) {
http://git-wip-us.apache.org/repos/asf/mesos/blob/0e331b0d/src/uri/fetcher.cpp
----------------------------------------------------------------------
diff --git a/src/uri/fetcher.cpp b/src/uri/fetcher.cpp
index aa9df5d..198bd29 100644
--- a/src/uri/fetcher.cpp
+++ b/src/uri/fetcher.cpp
@@ -46,8 +46,11 @@ Try<Owned<Fetcher>> create(const Option<Flags>& _flags)
hashmap<string, Creator> creators;
creators.put("curl", lambda::bind(&CurlFetcherPlugin::create, flags));
creators.put("file", lambda::bind(&CopyFetcherPlugin::create, flags));
+#ifndef __WINDOWS__
+ // TODO(dpravat): Enable `Hadoop` and `Docker` plugins. See MESOS-5473.
creators.put("hadoop", lambda::bind(&HadoopFetcherPlugin::create, flags));
creators.put("docker", lambda::bind(&DockerFetcherPlugin::create, flags));
+#endif // __WINDOWS__
hashmap<string, Owned<Fetcher::Plugin>> plugins;
http://git-wip-us.apache.org/repos/asf/mesos/blob/0e331b0d/src/uri/fetcher.hpp
----------------------------------------------------------------------
diff --git a/src/uri/fetcher.hpp b/src/uri/fetcher.hpp
index 8af2c91..949363e 100644
--- a/src/uri/fetcher.hpp
+++ b/src/uri/fetcher.hpp
@@ -39,9 +39,14 @@ namespace fetcher {
*/
class Flags :
public CopyFetcherPlugin::Flags,
+#ifdef __WINDOWS__
+ // TODO(dpravat): Add support for Hadoop and Docker plugins. See MESOS-5473.
+ public CurlFetcherPlugin::Flags {};
+#else
public CurlFetcherPlugin::Flags,
public HadoopFetcherPlugin::Flags,
public DockerFetcherPlugin::Flags {};
+#endif // __WINDOWS__
/**
http://git-wip-us.apache.org/repos/asf/mesos/blob/0e331b0d/src/uri/fetchers/copy.cpp
----------------------------------------------------------------------
diff --git a/src/uri/fetchers/copy.cpp b/src/uri/fetchers/copy.cpp
index 2180adf..f095ad6 100644
--- a/src/uri/fetchers/copy.cpp
+++ b/src/uri/fetchers/copy.cpp
@@ -14,7 +14,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#ifndef __WINDOWS__
#include <sys/wait.h>
+#endif // __WINDOWS__
#include <string>
#include <tuple>
http://git-wip-us.apache.org/repos/asf/mesos/blob/0e331b0d/src/uri/fetchers/curl.cpp
----------------------------------------------------------------------
diff --git a/src/uri/fetchers/curl.cpp b/src/uri/fetchers/curl.cpp
index c442062..cc3f9ee 100644
--- a/src/uri/fetchers/curl.cpp
+++ b/src/uri/fetchers/curl.cpp
@@ -14,7 +14,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#ifndef __WINDOWS__
#include <sys/wait.h>
+#endif // __WINDOWS__
#include <string>
#include <tuple>
http://git-wip-us.apache.org/repos/asf/mesos/blob/0e331b0d/src/zookeeper/group.cpp
----------------------------------------------------------------------
diff --git a/src/zookeeper/group.cpp b/src/zookeeper/group.cpp
index a9c6eae..7438ecc 100644
--- a/src/zookeeper/group.cpp
+++ b/src/zookeeper/group.cpp
@@ -39,6 +39,8 @@
#include <stout/strings.hpp>
#include <stout/utils.hpp>
+#include <stout/os/constants.hpp>
+
#include "logging/logging.hpp"
using namespace process;
@@ -653,7 +655,10 @@ Result<bool> GroupProcess::doCancel(const Group::Membership& membership)
{
CHECK_EQ(state, READY);
- string path = path::join(znode, zkBasename(membership));
+ string path = path::join(
+ znode,
+ zkBasename(membership),
+ os::POSIX_PATH_SEPARATOR);
LOG(INFO) << "Trying to remove '" << path << "' in ZooKeeper";
@@ -693,7 +698,10 @@ Result<Option<string>> GroupProcess::doData(
{
CHECK_EQ(state, READY);
- string path = path::join(znode, zkBasename(membership));
+ string path = path::join(
+ znode,
+ zkBasename(membership),
+ os::POSIX_PATH_SEPARATOR);
LOG(INFO) << "Trying to get '" << path << "' in ZooKeeper";