You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ti...@apache.org on 2016/05/30 18:01:50 UTC
[1/3] mesos git commit: Added --modules_dir flag to read module
manifests from a directory.
Repository: mesos
Updated Branches:
refs/heads/master 245ea1886 -> 41e94c5af
Added --modules_dir flag to read module manifests from a directory.
This allows the operator to use separate manifest JSON files for each
module. Previously, one had to merge all module manifest files into a
single JSON file before passing on to the master/agent.
Review: https://reviews.apache.org/r/47123/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/c7600b76
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/c7600b76
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/c7600b76
Branch: refs/heads/master
Commit: c7600b7632b32101b9ab998d8de7617163fc2773
Parents: 245ea18
Author: Kapil Arya <ka...@mesosphere.io>
Authored: Mon May 30 19:25:01 2016 +0200
Committer: Till Toenshoff <to...@me.com>
Committed: Mon May 30 19:25:01 2016 +0200
----------------------------------------------------------------------
src/master/flags.cpp | 15 +++++++++++++--
src/master/flags.hpp | 1 +
src/master/main.cpp | 12 ++++++++++++
src/module/manager.cpp | 45 ++++++++++++++++++++++++++++++++++++++++++++-
src/module/manager.hpp | 13 ++++++++++++-
src/sched/flags.hpp | 13 ++++++++++++-
src/sched/sched.cpp | 18 ++++++++++++++++++
src/slave/flags.cpp | 12 +++++++++++-
src/slave/flags.hpp | 1 +
src/slave/main.cpp | 12 ++++++++++++
src/tests/flags.hpp | 13 ++++++++++++-
src/tests/main.cpp | 15 +++++++++++++++
12 files changed, 163 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/c7600b76/src/master/flags.cpp
----------------------------------------------------------------------
diff --git a/src/master/flags.cpp b/src/master/flags.cpp
index ceb4bd4..9ec97a3 100644
--- a/src/master/flags.cpp
+++ b/src/master/flags.cpp
@@ -357,7 +357,7 @@ mesos::internal::master::Flags::Flags()
"If not set, offers do not timeout.");
// This help message for --modules flag is the same for
- // {master,slave,tests}/flags.hpp and should always be kept in
+ // {master,slave,sched,tests}/flags.[ch]pp and should always be kept in
// sync.
// TODO(karya): Remove the JSON example and add reference to the
// doc file explaining the --modules flag.
@@ -401,7 +401,18 @@ mesos::internal::master::Flags::Flags()
" ]\n"
" }\n"
" ]\n"
- "}");
+ "}\n\n"
+ "Cannot be used in conjunction with --modules_dir.\n");
+
+ // This help message for --modules_dir flag is the same for
+ // {master,slave,sched,tests}/flags.[ch]pp and should always be kept in
+ // sync.
+ add(&Flags::modulesDir,
+ "modules_dir",
+ "Directory path of the module manifest files.\n"
+ "The manifest files are processed in alphabetical order.\n"
+ "(See --modules for more information on module manifest files)\n"
+ "Cannot be used in conjunction with --modules.\n");
add(&Flags::authenticators,
"authenticators",
http://git-wip-us.apache.org/repos/asf/mesos/blob/c7600b76/src/master/flags.hpp
----------------------------------------------------------------------
diff --git a/src/master/flags.hpp b/src/master/flags.hpp
index d085096..735367f 100644
--- a/src/master/flags.hpp
+++ b/src/master/flags.hpp
@@ -74,6 +74,7 @@ public:
Option<RateLimits> rate_limits;
Option<Duration> offer_timeout;
Option<Modules> modules;
+ Option<std::string> modulesDir;
std::string authenticators;
std::string allocator;
Option<std::string> hooks;
http://git-wip-us.apache.org/repos/asf/mesos/blob/c7600b76/src/master/main.cpp
----------------------------------------------------------------------
diff --git a/src/master/main.cpp b/src/master/main.cpp
index 871cf2d..294b35d 100644
--- a/src/master/main.cpp
+++ b/src/master/main.cpp
@@ -253,6 +253,18 @@ int main(int argc, char** argv)
// Initialize modules. Note that since other subsystems may depend
// upon modules, we should initialize modules before anything else.
+ if (flags.modules.isSome() && flags.modulesDir.isSome()) {
+ EXIT(EXIT_FAILURE) <<
+ flags.usage("Only one of --modules or --modules_dir should be specified");
+ }
+
+ if (flags.modulesDir.isSome()) {
+ Try<Nothing> result = ModuleManager::load(flags.modulesDir.get());
+ if (result.isError()) {
+ EXIT(EXIT_FAILURE) << "Error loading modules: " << result.error();
+ }
+ }
+
if (flags.modules.isSome()) {
Try<Nothing> result = ModuleManager::load(flags.modules.get());
if (result.isError()) {
http://git-wip-us.apache.org/repos/asf/mesos/blob/c7600b76/src/module/manager.cpp
----------------------------------------------------------------------
diff --git a/src/module/manager.cpp b/src/module/manager.cpp
index 9f88ec3..725f085 100644
--- a/src/module/manager.cpp
+++ b/src/module/manager.cpp
@@ -30,6 +30,7 @@
#include <stout/stringify.hpp>
#include <stout/version.hpp>
+#include "common/parse.hpp"
#include "messages/messages.hpp"
#include "module/manager.hpp"
@@ -263,7 +264,7 @@ Try<Nothing> ModuleManager::verifyIdenticalModule(
}
-Try<Nothing> ModuleManager::load(const Modules& modules)
+Try<Nothing> ModuleManager::loadManifest(const Modules& modules)
{
synchronized (mutex) {
initialize();
@@ -347,3 +348,45 @@ Try<Nothing> ModuleManager::load(const Modules& modules)
return Nothing();
}
+
+
+// We load the module manifests sequentially in an alphabetical order. If an
+// error is encountered while processing a particular manifest, we do not load
+// the remaining manifests and exit with the appropriate error message.
+Try<Nothing> ModuleManager::load(const string& modulesDir)
+{
+ Try<list<string>> moduleManifests = os::ls(modulesDir);
+ if (moduleManifests.isError()) {
+ return Error(
+ "Error loading module manifests from '" + modulesDir + "' directory: " +
+ moduleManifests.error());
+ }
+
+ moduleManifests->sort();
+ foreach (const string& filename, moduleManifests.get()) {
+ const string filepath = path::join(modulesDir, filename);
+ VLOG(1) << "Processing module manifest from '" << filepath << "'";
+
+ Try<string> read = os::read(filepath);
+ if (read.isError()) {
+ return Error(
+ "Error reading module manifest file '" + filepath + "': " +
+ read.error());
+ }
+
+ Try<Modules> modules = flags::parse<Modules>(read.get());
+ if (modules.isError()) {
+ return Error(
+ "Error parsing module manifest file '" + filepath + "': " +
+ modules.error());
+ }
+
+ Try<Nothing> result = loadManifest(modules.get());
+ if (result.isError()) {
+ return Error(
+ "Error loading modules from '" + filepath + "': " + result.error());
+ }
+ }
+
+ return Nothing();
+}
http://git-wip-us.apache.org/repos/asf/mesos/blob/c7600b76/src/module/manager.hpp
----------------------------------------------------------------------
diff --git a/src/module/manager.hpp b/src/module/manager.hpp
index 9944af0..4a9e6a3 100644
--- a/src/module/manager.hpp
+++ b/src/module/manager.hpp
@@ -34,7 +34,9 @@
#include <stout/check.hpp>
#include <stout/dynamiclibrary.hpp>
#include <stout/hashmap.hpp>
+#include <stout/nothing.hpp>
#include <stout/synchronized.hpp>
+#include <stout/try.hpp>
#include "messages/messages.hpp"
@@ -62,7 +64,14 @@ public:
//
// NOTE: If loading fails at a particular library we don't unload
// all of the already loaded libraries.
- static Try<Nothing> load(const Modules& modules);
+ static Try<Nothing> load(const Modules& modules)
+ {
+ return loadManifest(modules);
+ }
+
+ // NOTE: If loading fails at a particular library we don't unload
+ // all of the already loaded libraries.
+ static Try<Nothing> load(const std::string& modulesDir);
// create() should be called only after load().
template <typename T>
@@ -140,6 +149,8 @@ public:
private:
static void initialize();
+ static Try<Nothing> loadManifest(const Modules& modules);
+
static Try<Nothing> verifyModule(
const std::string& moduleName,
const ModuleBase* moduleBase);
http://git-wip-us.apache.org/repos/asf/mesos/blob/c7600b76/src/sched/flags.hpp
----------------------------------------------------------------------
diff --git a/src/sched/flags.hpp b/src/sched/flags.hpp
index b4ca12b..989cebe 100644
--- a/src/sched/flags.hpp
+++ b/src/sched/flags.hpp
@@ -47,7 +47,7 @@ public:
DEFAULT_REGISTRATION_BACKOFF_FACTOR);
// This help message for --modules flag is the same for
- // {master,slave,tests,sched}/flags.hpp and should always be kept
+ // {master,slave,sched,tests}/flags.[ch]pp and should always be kept in
// in sync.
// TODO(karya): Remove the JSON example and add reference to the
// doc file explaining the --modules flag.
@@ -93,6 +93,16 @@ public:
" ]\n"
"}");
+ // This help message for --modules_dir flag is the same for
+ // {master,slave,sched,tests}/flags.[ch]pp and should always be kept in
+ // sync.
+ add(&Flags::modulesDir,
+ "modules_dir",
+ "Directory path of the module manifest files.\n"
+ "The manifest files are processed in alphabetical order.\n"
+ "(See --modules for more information on module manifest files)\n"
+ "Cannot be used in conjunction with --modules.\n");
+
add(&Flags::authenticatee,
"authenticatee",
"Authenticatee implementation to use when authenticating against the\n"
@@ -103,6 +113,7 @@ public:
Duration registration_backoff_factor;
Option<Modules> modules;
+ Option<std::string> modulesDir;
std::string authenticatee;
};
http://git-wip-us.apache.org/repos/asf/mesos/blob/c7600b76/src/sched/sched.cpp
----------------------------------------------------------------------
diff --git a/src/sched/sched.cpp b/src/sched/sched.cpp
index 8380e48..5eca265 100644
--- a/src/sched/sched.cpp
+++ b/src/sched/sched.cpp
@@ -1878,6 +1878,24 @@ Status MesosSchedulerDriver::start()
// Initialize modules. Note that since other subsystems may depend
// upon modules, we should initialize modules before anything else.
+ if (flags.modules.isSome() && flags.modulesDir.isSome()) {
+ status = DRIVER_ABORTED;
+ scheduler->error(
+ this,
+ "Only one of MESOS_MODULES or MESOS_MODULES_DIR should be specified");
+ return status;
+ }
+
+ if (flags.modulesDir.isSome()) {
+ Try<Nothing> result =
+ modules::ModuleManager::load(flags.modulesDir.get());
+ if (result.isError()) {
+ status = DRIVER_ABORTED;
+ scheduler->error(this, "Error loading modules: " + result.error());
+ return status;
+ }
+ }
+
if (flags.modules.isSome()) {
Try<Nothing> result = modules::ModuleManager::load(flags.modules.get());
if (result.isError()) {
http://git-wip-us.apache.org/repos/asf/mesos/blob/c7600b76/src/slave/flags.cpp
----------------------------------------------------------------------
diff --git a/src/slave/flags.cpp b/src/slave/flags.cpp
index d30f39c..f90beaf 100644
--- a/src/slave/flags.cpp
+++ b/src/slave/flags.cpp
@@ -687,7 +687,7 @@ mesos::internal::slave::Flags::Flags()
false);
// This help message for --modules flag is the same for
- // {master,slave,tests}/flags.hpp and should always be kept in
+ // {master,slave,sched,tests}/flags.[ch]pp and should always be kept in
// sync.
// TODO(karya): Remove the JSON example and add reference to the
// doc file explaining the --modules flag.
@@ -733,6 +733,16 @@ mesos::internal::slave::Flags::Flags()
" ]\n"
"}");
+ // This help message for --modules_dir flag is the same for
+ // {master,slave,sched,tests}/flags.[ch]pp and should always be kept in
+ // sync.
+ add(&Flags::modulesDir,
+ "modules_dir",
+ "Directory path of the module manifest files.\n"
+ "The manifest files are processed in alphabetical order.\n"
+ "(See --modules for more information on module manifest files)\n"
+ "Cannot be used in conjunction with --modules.\n");
+
add(&Flags::authenticatee,
"authenticatee",
"Authenticatee implementation to use when authenticating against the\n"
http://git-wip-us.apache.org/repos/asf/mesos/blob/c7600b76/src/slave/flags.hpp
----------------------------------------------------------------------
diff --git a/src/slave/flags.hpp b/src/slave/flags.hpp
index 3363412..17acf56 100644
--- a/src/slave/flags.hpp
+++ b/src/slave/flags.hpp
@@ -136,6 +136,7 @@ public:
Duration container_disk_watch_interval;
bool enforce_container_disk_quota;
Option<Modules> modules;
+ Option<std::string> modulesDir;
std::string authenticatee;
std::string authorizer;
std::string http_authenticators;
http://git-wip-us.apache.org/repos/asf/mesos/blob/c7600b76/src/slave/main.cpp
----------------------------------------------------------------------
diff --git a/src/slave/main.cpp b/src/slave/main.cpp
index f431e13..fddb05c 100644
--- a/src/slave/main.cpp
+++ b/src/slave/main.cpp
@@ -232,6 +232,18 @@ int main(int argc, char** argv)
// Initialize modules. Note that since other subsystems may depend
// upon modules, we should initialize modules before anything else.
+ if (flags.modules.isSome() && flags.modulesDir.isSome()) {
+ EXIT(EXIT_FAILURE) <<
+ flags.usage("Only one of --modules or --modules_dir should be specified");
+ }
+
+ if (flags.modulesDir.isSome()) {
+ Try<Nothing> result = ModuleManager::load(flags.modulesDir.get());
+ if (result.isError()) {
+ EXIT(EXIT_FAILURE) << "Error loading modules: " << result.error();
+ }
+ }
+
if (flags.modules.isSome()) {
Try<Nothing> result = ModuleManager::load(flags.modules.get());
if (result.isError()) {
http://git-wip-us.apache.org/repos/asf/mesos/blob/c7600b76/src/tests/flags.hpp
----------------------------------------------------------------------
diff --git a/src/tests/flags.hpp b/src/tests/flags.hpp
index ae232b1..3066399 100644
--- a/src/tests/flags.hpp
+++ b/src/tests/flags.hpp
@@ -84,7 +84,7 @@ public:
"/var/run/docker.sock");
// This help message for --modules flag is the same for
- // {master,slave,tests}/flags.hpp and should always be kept in
+ // {master,slave,sched,tests}/flags.[ch]pp and should always be kept in
// sync.
// TODO(karya): Remove the JSON example and add reference to the
// doc file explaining the --modules flag.
@@ -130,6 +130,16 @@ public:
" ]\n"
"}");
+ // This help message for --modules_dir flag is the same for
+ // {master,slave,sched,tests}/flags.[ch]pp and should always be kept in
+ // sync.
+ add(&Flags::modulesDir,
+ "modules_dir",
+ "Directory path of the module manifest files.\n"
+ "The manifest files are processed in alphabetical order.\n"
+ "(See --modules for more information on module manifest files)\n"
+ "Cannot be used in conjunction with --modules.\n");
+
// This help message is duplicated from slave/flags.hpp and
// should always be kept in sync with that.
add(&Flags::isolation,
@@ -158,6 +168,7 @@ public:
std::string docker;
std::string docker_socket;
Option<Modules> modules;
+ Option<std::string> modulesDir;
Option<std::string> isolation;
std::string authenticators;
};
http://git-wip-us.apache.org/repos/asf/mesos/blob/c7600b76/src/tests/main.cpp
----------------------------------------------------------------------
diff --git a/src/tests/main.cpp b/src/tests/main.cpp
index efec922..1425a04 100644
--- a/src/tests/main.cpp
+++ b/src/tests/main.cpp
@@ -29,6 +29,8 @@
#include "messages/messages.hpp" // For GOOGLE_PROTOBUF_VERIFY_VERSION.
+#include "module/manager.hpp"
+
#include "tests/environment.hpp"
#include "tests/flags.hpp"
#include "tests/mesos.hpp"
@@ -65,6 +67,19 @@ int main(int argc, char** argv)
}
// Initialize Modules.
+ if (flags.modules.isSome() && flags.modulesDir.isSome()) {
+ EXIT(EXIT_FAILURE) <<
+ flags.usage("Only one of --modules or --modules_dir should be specified");
+ }
+
+ if (flags.modulesDir.isSome()) {
+ Try<Nothing> result =
+ mesos::modules::ModuleManager::load(flags.modulesDir.get());
+ if (result.isError()) {
+ EXIT(EXIT_FAILURE) << "Error loading modules: " << result.error();
+ }
+ }
+
Try<Nothing> result = tests::initModules(flags.modules);
if (result.isError()) {
cerr << "Error initializing modules: " << result.error() << endl;
[2/3] mesos git commit: Added test for --modules_dir flag.
Posted by ti...@apache.org.
Added test for --modules_dir flag.
Review: https://reviews.apache.org/r/47905/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/4dfdc875
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/4dfdc875
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/4dfdc875
Branch: refs/heads/master
Commit: 4dfdc8757946438610356476ba3f556a4c0a6b6f
Parents: c7600b7
Author: Kapil Arya <ka...@mesosphere.io>
Authored: Mon May 30 19:25:13 2016 +0200
Committer: Till Toenshoff <to...@me.com>
Committed: Mon May 30 19:59:57 2016 +0200
----------------------------------------------------------------------
src/tests/module_tests.cpp | 38 ++++++++++++++++++++++++++++++++++++++
1 file changed, 38 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfdc875/src/tests/module_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/module_tests.cpp b/src/tests/module_tests.cpp
index 950d8e4..54a22e3 100644
--- a/src/tests/module_tests.cpp
+++ b/src/tests/module_tests.cpp
@@ -16,12 +16,16 @@
#include <mesos/module.hpp>
+#include <mesos/module/hook.hpp>
#include <mesos/module/isolator.hpp>
#include <mesos/module/module.hpp>
#include <mesos/slave/isolator.hpp>
#include <stout/dynamiclibrary.hpp>
+#include <stout/os.hpp>
+#include <stout/path.hpp>
+#include <stout/stringify.hpp>
#include "common/parse.hpp"
#include "examples/test_module.hpp"
@@ -193,6 +197,40 @@ TEST_F(ModuleTest, ExampleModuleLoadTest)
}
+// Test that we can load module manifests from modules-dir.
+TEST_F(ModuleTest, ModulesDirTest)
+{
+ string modulesDir = path::join(os::getcwd(), "modules_dir");
+ ASSERT_SOME(os::mkdir(modulesDir));
+
+ // Create a JSON file for the example module.
+ EXPECT_SOME(os::write(
+ path::join(modulesDir, "default_module.json"),
+ stringify(JSON::protobuf(defaultModules))));
+
+ // Let's also create another JSON file for the example module.
+ EXPECT_SOME(os::write(
+ path::join(modulesDir, "dup_default_module.json"),
+ stringify(JSON::protobuf(defaultModules))));
+
+ // Create a JSON for the 'TestHook' module.
+ Modules extraModules = getModules("testhook", "org_apache_mesos_TestHook");
+
+ EXPECT_SOME(os::write(
+ path::join(modulesDir, "extra_module.json"),
+ stringify(JSON::protobuf(extraModules))));
+
+ // Now load modules using the modules directory.
+ EXPECT_SOME(ModuleManager::load(modulesDir));
+
+ EXPECT_TRUE(ModuleManager::contains<TestModule>(DEFAULT_MODULE_NAME));
+ EXPECT_TRUE(ModuleManager::contains<Hook>("org_apache_mesos_TestHook"));
+
+ // Perform cleanup.
+ EXPECT_SOME(os::rmdir(modulesDir));
+}
+
+
// Test passing parameter without value.
TEST_F(ModuleTest, ParameterWithoutValue)
{
[3/3] mesos git commit: Added/updated documentation for --modules and
--modules_dir flags.
Posted by ti...@apache.org.
Added/updated documentation for --modules and --modules_dir flags.
Review: https://reviews.apache.org/r/47906/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/41e94c5a
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/41e94c5a
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/41e94c5a
Branch: refs/heads/master
Commit: 41e94c5af2dd018233893e469aa90d87923b6c8a
Parents: 4dfdc87
Author: Kapil Arya <ka...@mesosphere.io>
Authored: Mon May 30 19:25:24 2016 +0200
Committer: Till Toenshoff <to...@me.com>
Committed: Mon May 30 19:59:57 2016 +0200
----------------------------------------------------------------------
docs/modules.md | 23 ++++++++++++++++-------
1 file changed, 16 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/41e94c5a/docs/modules.md
----------------------------------------------------------------------
diff --git a/docs/modules.md b/docs/modules.md
index cf80e1c..aa5e50c 100644
--- a/docs/modules.md
+++ b/docs/modules.md
@@ -29,19 +29,28 @@ scripting languages without forcing those dependencies into the project.
Finally, modules provide an easy way for third parties to easily extend Mesos
without having to know all the internal details.
+## Where can Mesos modules be used?
+Modules can be specified for master, agent and tests. Modules can also be used
+with schedulers that link against libmesos. Currently, modules can not be
+used with executors.
## <a name="Invoking"></a>Invoking Mesos modules
+There are two ways to specify a list of modules to be loaded and be available to
+the internal subsystems.
-The command-line flag `--modules` is available for Mesos master, agent, and
-tests to specify a list of modules to be loaded and be available to the internal
-subsystems.
-
-Use `--modules=filepath` to specify the list of modules via a
-file containing a JSON-formatted string. 'filepath' can be
-of the form 'file:///path/to/file' or '/path/to/file'.
+* `--modules=filepath` to specify the list of modules via a manifest file
+ containing a JSON-formatted string. 'filepath' can be of the form
+ 'file:///path/to/file' or '/path/to/file'.
Use `--modules="{...}"` to specify the list of modules inline.
+* `--modules_dir=dirpath`to specify a directory name that contains several
+ module manifest files. Each file must contain a valid JSON-formatted
+ string containing module description. The module manifest files are processed
+ in alphabetical order.
+
+NOTE: Only one of `--modules` or `--modules_dir` flag can be used at time.
+
### Example JSON strings: