You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by jo...@apache.org on 2015/08/31 19:19:24 UTC
[1/8] mesos git commit: Maintenance primitives: Added Unavailability
and InverseOffer protobufs.
Repository: mesos
Updated Branches:
refs/heads/master 067034047 -> 09e4ba2fa
Maintenance primitives: Added Unavailability and InverseOffer protobufs.
MESOS-2061: Add Unavailability and InverseOffer protobufs declarations.
MESOS-2066: Add the Unavailability field to Offers.
Also copied to v1 API.
No integration with other components (that part is tracked in separate
JIRAs, see MESOS-1474).
Review: https://reviews.apache.org/r/36321
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/7d7f42f0
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/7d7f42f0
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/7d7f42f0
Branch: refs/heads/master
Commit: 7d7f42f08ca74026ab2db92583b342c88e405e58
Parents: 0670340
Author: Joseph Wu <jo...@mesosphere.io>
Authored: Sun Aug 30 13:54:54 2015 -0400
Committer: Joris Van Remoortere <jo...@gmail.com>
Committed: Mon Aug 31 13:09:09 2015 -0400
----------------------------------------------------------------------
include/mesos/mesos.proto | 80 +++++++++++++++++++++++++++++++++++++++
include/mesos/v1/mesos.proto | 80 +++++++++++++++++++++++++++++++++++++++
2 files changed, 160 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/7d7f42f0/include/mesos/mesos.proto
----------------------------------------------------------------------
diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto
index c40a09b..1e07159 100644
--- a/include/mesos/mesos.proto
+++ b/include/mesos/mesos.proto
@@ -138,6 +138,23 @@ message URL {
/**
+ * Represents an interval, from a given start time over a given duration.
+ * This interval pertains to an unavailability event, such as maintenance,
+ * and is not a generic interval.
+ */
+message Unavailability {
+ required TimeInfo start = 1;
+
+ // When added to `start`, this represents the end of the interval.
+ // If unspecified, the duration is assumed to be infinite.
+ optional DurationInfo duration = 2;
+
+ // TODO(josephw): Add additional fields for expressing the purpose and
+ // urgency of the unavailability event.
+}
+
+
+/**
* Describes a framework.
*/
message FrameworkInfo {
@@ -854,6 +871,18 @@ message Offer {
repeated Attribute attributes = 7;
repeated ExecutorID executor_ids = 6;
+ // Signifies that the resources in this Offer may be unavailable during
+ // the given interval. Any tasks launched using these resources may be
+ // killed when the interval arrives. For example, these resources may be
+ // part of a planned maintenance schedule.
+ //
+ // This field only provides information about a planned unavailability.
+ // The unavailability interval may not necessarily start at exactly this
+ // interval, nor last for exactly the duration of this interval.
+ // The unavailability may also be forever! See comments in
+ // `Unavailability` for more details.
+ optional Unavailability unavailability = 9;
+
// Defines an operation that can be performed against offers.
message Operation {
enum Type {
@@ -895,6 +924,57 @@ message Offer {
/**
+ * A request to return some resources occupied by a framework.
+ */
+message InverseOffer {
+ // This is the same OfferID as found in normal offers, which allows
+ // re-use of some of the OfferID-only messages.
+ required OfferID id = 1;
+
+ // URL for reaching the slave running on the host. This enables some
+ // optimizations as described in MESOS-3012, such as allowing the
+ // scheduler driver to bypass the master and talk directly with a slave.
+ optional URL url = 2;
+
+ // The framework that should release its resources.
+ // If no specifics are provided (i.e. which slave), all the framework's
+ // resources are requested back.
+ required FrameworkID framework_id = 3;
+
+ // Specified if the resources need to be released from a particular slave.
+ // All the framework's resources on this slave are requested back,
+ // unless further qualified by the `resources` field.
+ optional SlaveID slave_id = 4;
+
+ // This InverseOffer represents a planned unavailability event in the
+ // specified interval. Any tasks running on the given framework or slave
+ // may be killed when the interval arrives. Therefore, frameworks should
+ // aim to gracefully terminate tasks prior to the arrival of the interval.
+ //
+ // For reserved resources, the resources are expected to be returned to the
+ // framework after the unavailability interval. This is an expectation,
+ // not a guarantee. For example, if the unavailiability duration is not set,
+ // the resources may be removed permenantly.
+ //
+ // For other resources, there is no guarantee that requested resources will
+ // be returned after the unavailiability interval. The allocator has no
+ // obligation to re-offer these resources to the prior framework after
+ // the unavailability.
+ required Unavailability unavailability = 5;
+
+ // A list of resources being requested back from the framework,
+ // on the slave identified by `slave_id`. If no resources are specified
+ // then all resources are being requested back. For the purpose of
+ // maintenance, this field is always empty (maintenance always requests
+ // all resources back).
+ repeated Resource resources = 6;
+
+ // TODO(josephw): Add additional options for narrowing down the resources
+ // being requested back. Such as specific executors, tasks, etc.
+}
+
+
+/**
* Describes a task. Passed from the scheduler all the way to an
* executor (see SchedulerDriver::launchTasks and
* Executor::launchTask). Either ExecutorInfo or CommandInfo should be set.
http://git-wip-us.apache.org/repos/asf/mesos/blob/7d7f42f0/include/mesos/v1/mesos.proto
----------------------------------------------------------------------
diff --git a/include/mesos/v1/mesos.proto b/include/mesos/v1/mesos.proto
index ee15b9a..3ac1582 100644
--- a/include/mesos/v1/mesos.proto
+++ b/include/mesos/v1/mesos.proto
@@ -138,6 +138,23 @@ message URL {
/**
+ * Represents an interval, from a given start time over a given duration.
+ * This interval pertains to an unavailability event, such as maintenance,
+ * and is not a generic interval.
+ */
+message Unavailability {
+ required TimeInfo start = 1;
+
+ // When added to `start`, this represents the end of the interval.
+ // If unspecified, the duration is assumed to be infinite.
+ optional DurationInfo duration = 2;
+
+ // TODO(josephw): Add additional fields for expressing the purpose and
+ // urgency of the unavailability event.
+}
+
+
+/**
* Describes a framework.
*/
message FrameworkInfo {
@@ -847,6 +864,18 @@ message Offer {
repeated Attribute attributes = 7;
repeated ExecutorID executor_ids = 6;
+ // Signifies that the resources in this Offer may be unavailable during
+ // the given interval. Any tasks launched using these resources may be
+ // killed when the interval arrives. For example, these resources may be
+ // part of a planned maintenance schedule.
+ //
+ // This field only provides information about a planned unavailability.
+ // The unavailability interval may not necessarily start at exactly this
+ // interval, nor last for exactly the duration of this interval.
+ // The unavailability may also be forever! See comments in
+ // `Unavailability` for more details.
+ optional Unavailability unavailability = 9;
+
// Defines an operation that can be performed against offers.
message Operation {
enum Type {
@@ -888,6 +917,57 @@ message Offer {
/**
+ * A request to return some resources occupied by a framework.
+ */
+message InverseOffer {
+ // This is the same OfferID as found in normal offers, which allows
+ // re-use of some of the OfferID-only messages.
+ required OfferID id = 1;
+
+ // URL for reaching the agent running on the host. This enables some
+ // optimizations as described in MESOS-3012, such as allowing the
+ // scheduler driver to bypass the master and talk directly with a agent.
+ optional URL url = 2;
+
+ // The framework that should release its resources.
+ // If no specifics are provided (i.e. which agent), all the framework's
+ // resources are requested back.
+ required FrameworkID framework_id = 3;
+
+ // Specified if the resources need to be released from a particular agent.
+ // All the framework's resources on this agent are requested back,
+ // unless further qualified by the `resources` field.
+ optional AgentID agent_id = 4;
+
+ // This InverseOffer represents a planned unavailability event in the
+ // specified interval. Any tasks running on the given framework or agent
+ // may be killed when the interval arrives. Therefore, frameworks should
+ // aim to gracefully terminate tasks prior to the arrival of the interval.
+ //
+ // For reserved resources, the resources are expected to be returned to the
+ // framework after the unavailability interval. This is an expectation,
+ // not a guarantee. For example, if the unavailiability duration is not set,
+ // the resources may be removed permenantly.
+ //
+ // For other resources, there is no guarantee that requested resources will
+ // be returned after the unavailiability interval. The allocator has no
+ // obligation to re-offer these resources to the prior framework after
+ // the unavailability.
+ required Unavailability unavailability = 5;
+
+ // A list of resources being requested back from the framework,
+ // on the agent identified by `agent_id`. If no resources are specified
+ // then all resources are being requested back. For the purpose of
+ // maintenance, this field is always empty (maintenance always requests
+ // all resources back).
+ repeated Resource resources = 6;
+
+ // TODO(josephw): Add additional options for narrowing down the resources
+ // being requested back. Such as specific executors, tasks, etc.
+}
+
+
+/**
* Describes a task. Passed from the scheduler all the way to an
* executor (see SchedulerDriver::launchTasks and
* Executor::launchTask). Either ExecutorInfo or CommandInfo should be set.
[7/8] mesos git commit: Maintenance Primitives: Added machine UP
endpoint.
Posted by jo...@apache.org.
Maintenance Primitives: Added machine UP endpoint.
Endpoint: /machine/up
Transitions agents back into UP mode.
Registry operation = maintenance::StopMaintenance
Sets the list of machines back to UP mode. Removes those machines
from the maintenance schedule.
Review: https://reviews.apache.org/r/37362
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/bf4ca549
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/bf4ca549
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/bf4ca549
Branch: refs/heads/master
Commit: bf4ca5497fbed88ec54fd25f962c5eb1cee3b48c
Parents: de231ed
Author: Joseph Wu <jo...@mesosphere.io>
Authored: Sun Aug 30 13:56:33 2015 -0400
Committer: Joris Van Remoortere <jo...@gmail.com>
Committed: Mon Aug 31 13:15:16 2015 -0400
----------------------------------------------------------------------
src/master/http.cpp | 103 +++++++++++++++++++++++
src/master/maintenance.cpp | 55 +++++++++++++
src/master/maintenance.hpp | 23 ++++++
src/master/master.cpp | 6 ++
src/master/master.hpp | 5 ++
src/tests/master_maintenance_tests.cpp | 121 +++++++++++++++++++++++++++-
src/tests/registrar_tests.cpp | 79 ++++++++++++++++++
7 files changed, 391 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/bf4ca549/src/master/http.cpp
----------------------------------------------------------------------
diff --git a/src/master/http.cpp b/src/master/http.cpp
index 11e786d..6fad959 100644
--- a/src/master/http.cpp
+++ b/src/master/http.cpp
@@ -93,6 +93,7 @@ using process::http::UnsupportedMediaType;
using process::metrics::internal::MetricsProcess;
+using std::list;
using std::map;
using std::string;
using std::vector;
@@ -1548,6 +1549,108 @@ Future<Response> Master::Http::machineDown(const Request& request) const
}
+// /master/maintenance/start endpoint help.
+const string Master::Http::MACHINE_UP_HELP = HELP(
+ TLDR(
+ "Brings a set of machines back up."),
+ USAGE(
+ "/master/machine/up"),
+ DESCRIPTION(
+ "POST: Validates the request body as JSON and transitions",
+ " the list of machines into UP mode. This also removes",
+ " the list of machines from the maintenance schedule."));
+
+
+// /master/machine/up endpoint handler.
+Future<Response> Master::Http::machineUp(const Request& request) const
+{
+ if (request.method != "POST") {
+ return BadRequest("Expecting POST, got '" + request.method + "'");
+ }
+
+ // Parse the POST body as JSON.
+ Try<JSON::Object> jsonIds = JSON::parse<JSON::Object>(request.body);
+ if (jsonIds.isError()) {
+ return BadRequest(jsonIds.error());
+ }
+
+ // Convert the machines to a protobuf.
+ Try<MachineIDs> protoIds =
+ ::protobuf::parse<MachineIDs>(jsonIds.get());
+
+ if (protoIds.isError()) {
+ return BadRequest(protoIds.error());
+ }
+
+ // Validate every machine in the list.
+ MachineIDs ids = protoIds.get();
+ Try<Nothing> isValid = maintenance::validation::machines(ids);
+ if (isValid.isError()) {
+ return BadRequest(isValid.error());
+ }
+
+ // Check that all machines are part of a maintenance schedule.
+ foreach (const MachineID& id, ids.values()) {
+ if (!master->machineInfos.contains(id)) {
+ return BadRequest(
+ "Machine '" + id.DebugString() +
+ "' is not part of a maintenance schedule");
+ }
+
+ if (master->machineInfos[id].mode() != MachineInfo::DOWN) {
+ return BadRequest(
+ "Machine '" + id.DebugString() +
+ "' is not in DOWN mode and cannot be brought up");
+ }
+ }
+
+ return master->registrar->apply(Owned<Operation>(
+ new maintenance::StopMaintenance(ids)))
+ .then(defer(master->self(), [=](bool result) -> Future<Response> {
+ // See the top comment in "master/maintenance.hpp" for why this check
+ // is here, and is appropriate.
+ CHECK(result);
+
+ // Update the master's local state with the reactivated machines.
+ hashset<MachineID> updated;
+ foreach (const MachineID& id, ids.values()) {
+ master->machineInfos.erase(id);
+ updated.insert(id);
+ }
+
+ // Delete the machines from the schedule.
+ for (list<mesos::maintenance::Schedule>::iterator schedule =
+ master->maintenance.schedules.begin();
+ schedule != master->maintenance.schedules.end();) {
+ for (int j = schedule->windows().size() - 1; j >= 0; j--) {
+ mesos::maintenance::Window* window = schedule->mutable_windows(j);
+
+ // Delete individual machines.
+ for (int k = window->machine_ids().size() - 1; k >= 0; k--) {
+ if (updated.contains(window->machine_ids(k))) {
+ window->mutable_machine_ids()->DeleteSubrange(k, 1);
+ }
+ }
+
+ // If the resulting window is empty, delete it.
+ if (window->machine_ids().size() == 0) {
+ schedule->mutable_windows()->DeleteSubrange(j, 1);
+ }
+ }
+
+ // If the resulting schedule is empty, delete it.
+ if (schedule->windows().size() == 0) {
+ schedule = master->maintenance.schedules.erase(schedule);
+ } else {
+ ++schedule;
+ }
+ }
+
+ return OK();
+ }));
+}
+
+
Result<Credential> Master::Http::authenticate(const Request& request) const
{
// By default, assume everyone is authenticated if no credentials
http://git-wip-us.apache.org/repos/asf/mesos/blob/bf4ca549/src/master/maintenance.cpp
----------------------------------------------------------------------
diff --git a/src/master/maintenance.cpp b/src/master/maintenance.cpp
index 859cef9..277dd82 100644
--- a/src/master/maintenance.cpp
+++ b/src/master/maintenance.cpp
@@ -142,6 +142,61 @@ Try<bool> StartMaintenance::perform(
}
+StopMaintenance::StopMaintenance(
+ const MachineIDs& _ids)
+{
+ foreach (const MachineID& id, _ids.values()) {
+ ids.insert(id);
+ }
+}
+
+
+Try<bool> StopMaintenance::perform(
+ Registry* registry,
+ hashset<SlaveID>* slaveIDs,
+ bool strict)
+{
+ // Delete the machine info entry of all targeted machines.
+ // i.e. Transition them into `UP` mode.
+ bool changed = false;
+ for (int i = registry->machines().machines().size() - 1; i >= 0; i--) {
+ if (ids.contains(registry->machines().machines(i).info().id())) {
+ registry->mutable_machines()->mutable_machines()->DeleteSubrange(i, 1);
+
+ changed = true; // Mutation.
+ }
+ }
+
+ // Delete the machines from the schedule.
+ for (int i = registry->schedules().size() - 1; i >= 0; i--) {
+ maintenance::Schedule* schedule = registry->mutable_schedules(i);
+
+ for (int j = schedule->windows().size() - 1; j >= 0; j--) {
+ maintenance::Window* window = schedule->mutable_windows(j);
+
+ // Delete individual machines.
+ for (int k = window->machine_ids().size() - 1; k >= 0; k--) {
+ if (ids.contains(window->machine_ids(k))) {
+ window->mutable_machine_ids()->DeleteSubrange(k, 1);
+ }
+ }
+
+ // If the resulting window is empty, delete it.
+ if (window->machine_ids().size() == 0) {
+ schedule->mutable_windows()->DeleteSubrange(j, 1);
+ }
+ }
+
+ // If the resulting schedule is empty, delete it.
+ if (schedule->windows().size() == 0) {
+ registry->mutable_schedules()->DeleteSubrange(i, 1);
+ }
+ }
+
+ return changed;
+}
+
+
namespace validation {
Try<Nothing> schedule(
http://git-wip-us.apache.org/repos/asf/mesos/blob/bf4ca549/src/master/maintenance.hpp
----------------------------------------------------------------------
diff --git a/src/master/maintenance.hpp b/src/master/maintenance.hpp
index 8e6cb9c..bebaeb2 100644
--- a/src/master/maintenance.hpp
+++ b/src/master/maintenance.hpp
@@ -89,6 +89,29 @@ private:
};
+/**
+ * Transitions a group of machines from `DOWN` mode into `UP` mode.
+ * All machines must be in `DOWN` mode and must be part of a maintenance
+ * schedule prior to executing this operation. The machines will be
+ * removed from the maintenance schedule.
+ */
+class StopMaintenance : public Operation
+{
+public:
+ explicit StopMaintenance(
+ const MachineIDs& _ids);
+
+protected:
+ Try<bool> perform(
+ Registry* registry,
+ hashset<SlaveID>* slaveIDs,
+ bool strict);
+
+private:
+ hashset<MachineID> ids;
+};
+
+
namespace validation {
/**
http://git-wip-us.apache.org/repos/asf/mesos/blob/bf4ca549/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 06e283d..cd1b386 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -827,6 +827,12 @@ void Master::initialize()
Http::log(request);
return http.machineDown(request);
});
+ route("/machine/up",
+ Http::MACHINE_UP_HELP,
+ [http](const process::http::Request& request) {
+ Http::log(request);
+ return http.machineUp(request);
+ });
// Provide HTTP assets from a "webui" directory. This is either
// specified via flags (which is necessary for running out of the
http://git-wip-us.apache.org/repos/asf/mesos/blob/bf4ca549/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index c7e96db..68be718 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -840,6 +840,10 @@ private:
process::Future<process::http::Response> machineDown(
const process::http::Request& request) const;
+ // /master/machine/up
+ process::Future<process::http::Response> machineUp(
+ const process::http::Request& request) const;
+
const static std::string SCHEDULER_HELP;
const static std::string HEALTH_HELP;
const static std::string OBSERVE_HELP;
@@ -852,6 +856,7 @@ private:
const static std::string TASKS_HELP;
const static std::string MAINTENANCE_SCHEDULE_HELP;
const static std::string MACHINE_DOWN_HELP;
+ const static std::string MACHINE_UP_HELP;
private:
// Helper for doing authentication, returns the credential used if
http://git-wip-us.apache.org/repos/asf/mesos/blob/bf4ca549/src/tests/master_maintenance_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/master_maintenance_tests.cpp b/src/tests/master_maintenance_tests.cpp
index 67301cc..c3acd7a 100644
--- a/src/tests/master_maintenance_tests.cpp
+++ b/src/tests/master_maintenance_tests.cpp
@@ -18,6 +18,8 @@
#include <string>
+#include <mesos/maintenance/maintenance.hpp>
+
#include <process/clock.hpp>
#include <process/future.hpp>
#include <process/http.hpp>
@@ -28,6 +30,7 @@
#include <stout/json.hpp>
#include <stout/net.hpp>
#include <stout/option.hpp>
+#include <stout/protobuf.hpp>
#include <stout/strings.hpp>
#include <stout/stringify.hpp>
#include <stout/try.hpp>
@@ -325,8 +328,124 @@ TEST_F(MasterMaintenanceTest, BringDownMachines)
"machine/down",
headers,
stringify(JSON::Protobuf(machines)));
-
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+}
+
+
+// Posts valid and invalid machines to the maintenance stop endpoint.
+TEST_F(MasterMaintenanceTest, BringUpMachines)
+{
+ // Set up a master.
+ Try<PID<Master>> master = StartMaster();
+ ASSERT_SOME(master);
+
+ // Try to bring up an unscheduled machine.
+ MachineIDs machines = createMachineList({machine1, machine2});
+ Future<Response> response = process::http::post(
+ master.get(),
+ "machine/up",
+ headers,
+ stringify(JSON::Protobuf(machines)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(BadRequest().status, response);
+
+ // Post a valid schedule with three machines.
+ maintenance::Schedule schedule = createSchedule({
+ createWindow({machine1, machine2}, unavailability),
+ createWindow({machine3}, unavailability)});
+
+ response = process::http::post(
+ master.get(),
+ "maintenance/schedule",
+ headers,
+ stringify(JSON::Protobuf(schedule)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+
+ // Try to bring up a non-down machine.
+ machines = createMachineList({machine1, machine2});
+ response = process::http::post(
+ master.get(),
+ "machine/up",
+ headers,
+ stringify(JSON::Protobuf(machines)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(BadRequest().status, response);
+
+ // Down machine3.
+ machines = createMachineList({machine3});
+ response = process::http::post(
+ master.get(),
+ "machine/down",
+ headers,
+ stringify(JSON::Protobuf(machines)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+
+ // Up machine3.
+ response = process::http::post(
+ master.get(),
+ "machine/up",
+ headers,
+ stringify(JSON::Protobuf(machines)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+
+ // Get the maintenance schedule.
+ response = process::http::get(
+ master.get(),
+ "maintenance/schedule");
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+
+ // Check that only one maintenance window remains.
+ Try<JSON::Object> masterSchedule_ =
+ JSON::parse<JSON::Object>(response.get().body);
+
+ ASSERT_SOME(masterSchedule_);
+ Try<mesos::maintenance::Schedule> masterSchedule =
+ ::protobuf::parse<mesos::maintenance::Schedule>(masterSchedule_.get());
+
+ ASSERT_SOME(masterSchedule);
+ ASSERT_EQ(1, masterSchedule.get().windows().size());
+ ASSERT_EQ(2, masterSchedule.get().windows(0).machine_ids().size());
+
+ // Down the other machines.
+ machines = createMachineList({machine1, machine2});
+ response = process::http::post(
+ master.get(),
+ "machine/down",
+ headers,
+ stringify(JSON::Protobuf(machines)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+
+ // Up the other machines.
+ response = process::http::post(
+ master.get(),
+ "machine/up",
+ headers,
+ stringify(JSON::Protobuf(machines)));
+
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+
+ // Get the maintenance schedule again.
+ response = process::http::get(
+ master.get(),
+ "maintenance/schedule");
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+
+ // Check that the schedule is empty.
+ masterSchedule_ = JSON::parse<JSON::Object>(response.get().body);
+
+ ASSERT_SOME(masterSchedule_);
+ masterSchedule =
+ ::protobuf::parse<mesos::maintenance::Schedule>(masterSchedule_.get());
+
+ ASSERT_SOME(masterSchedule);
+ ASSERT_EQ(0, masterSchedule.get().windows().size());
}
} // namespace tests {
http://git-wip-us.apache.org/repos/asf/mesos/blob/bf4ca549/src/tests/registrar_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/registrar_tests.cpp b/src/tests/registrar_tests.cpp
index 733a2cd..aa49c86 100644
--- a/src/tests/registrar_tests.cpp
+++ b/src/tests/registrar_tests.cpp
@@ -585,6 +585,85 @@ TEST_P(RegistrarTest, StartMaintenance)
}
+// Creates a schedule and properly starts and stops maintenance.
+TEST_P(RegistrarTest, StopMaintenance)
+{
+ // Machine definitions used in this test.
+ MachineID machine1;
+ machine1.set_ip("0.0.0.1");
+
+ MachineID machine2;
+ machine2.set_hostname("2");
+
+ MachineID machine3;
+ machine3.set_hostname("3");
+ machine3.set_ip("0.0.0.3");
+
+ Unavailability unavailability = createUnavailability(Clock::now());
+
+ {
+ // Prepare the registrar.
+ Registrar registrar(flags, state);
+ AWAIT_READY(registrar.recover(master));
+
+ // Schdule three machines for maintenance.
+ maintenance::Schedule schedule = createSchedule({
+ createWindow({machine1, machine2}, unavailability),
+ createWindow({machine3}, unavailability)});
+
+ AWAIT_READY(registrar.apply(
+ Owned<Operation>(new UpdateSchedule(schedule))));
+
+ // Transition machine three into `DOWN` mode.
+ MachineIDs machines = createMachineList({machine3});
+ AWAIT_READY(registrar.apply(
+ Owned<Operation>(new StartMaintenance(machines))));
+
+ // Transition machine three into `UP` mode.
+ AWAIT_READY(registrar.apply(
+ Owned<Operation>(new StopMaintenance(machines))));
+ }
+
+ {
+ // Check that machine three and the window were removed.
+ Registrar registrar(flags, state);
+ Future<Registry> registry = registrar.recover(master);
+ AWAIT_READY(registry);
+
+ EXPECT_EQ(1, registry.get().schedules().size());
+ EXPECT_EQ(1, registry.get().schedules(0).windows().size());
+ EXPECT_EQ(2, registry.get().schedules(0).windows(0).machine_ids().size());
+ EXPECT_EQ(2, registry.get().machines().machines().size());
+ EXPECT_EQ(
+ MachineInfo::DRAINING,
+ registry.get().machines().machines(0).info().mode());
+
+ EXPECT_EQ(
+ MachineInfo::DRAINING,
+ registry.get().machines().machines(1).info().mode());
+
+ // Transition machine one and two into `DOWN` mode.
+ MachineIDs machines = createMachineList({machine1, machine2});
+ AWAIT_READY(registrar.apply(
+ Owned<Operation>(new StartMaintenance(machines))));
+
+ // Transition all machines into `UP` mode.
+ AWAIT_READY(registrar.apply(
+ Owned<Operation>(new StopMaintenance(machines))));
+ }
+
+ {
+ // Check that the schedule is now empty.
+ Registrar registrar(flags, state);
+ Future<Registry> registry = registrar.recover(master);
+ AWAIT_READY(registry);
+
+ EXPECT_EQ(0, registry.get().schedules().size());
+ EXPECT_EQ(0, registry.get().machines().machines().size());
+ }
+}
+
+
TEST_P(RegistrarTest, Bootstrap)
{
// Run 1 readmits a slave that is not present.
[5/8] mesos git commit: Maintenance Primitives: Added maintenance
schedule endpoint.
Posted by jo...@apache.org.
Maintenance Primitives: Added maintenance schedule endpoint.
Endpoint: /maintenance/schedule
Registry operation = maintenance::UpdateSchedule
Replaces the schedule with the given one. Also sets all scheduled
machines into Draining mode.
Review: https://reviews.apache.org/r/37325
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/3b0fe5cc
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/3b0fe5cc
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/3b0fe5cc
Branch: refs/heads/master
Commit: 3b0fe5cc047f9b96dcd96771fa8945483ce92e72
Parents: 8e43aba
Author: Joseph Wu <jo...@mesosphere.io>
Authored: Sun Aug 30 13:56:02 2015 -0400
Committer: Joris Van Remoortere <jo...@gmail.com>
Committed: Mon Aug 31 13:09:48 2015 -0400
----------------------------------------------------------------------
src/Makefile.am | 1 +
src/master/http.cpp | 126 +++++++++++++++
src/master/maintenance.cpp | 76 +++++++++
src/master/maintenance.hpp | 31 ++++
src/master/master.cpp | 6 +
src/master/master.hpp | 5 +
src/tests/master_maintenance_tests.cpp | 239 ++++++++++++++++++++++++++++
src/tests/registrar_tests.cpp | 184 +++++++++++++++++++++
8 files changed, 668 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/3b0fe5cc/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 1b07af4..7b4d9f6 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -1639,6 +1639,7 @@ mesos_tests_SOURCES = \
tests/master_allocator_tests.cpp \
tests/master_authorization_tests.cpp \
tests/master_contender_detector_tests.cpp \
+ tests/master_maintenance_tests.cpp \
tests/master_slave_reconciliation_tests.cpp \
tests/master_tests.cpp \
tests/master_validation_tests.cpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/3b0fe5cc/src/master/http.cpp
----------------------------------------------------------------------
diff --git a/src/master/http.cpp b/src/master/http.cpp
index 37d76ee..44178d8 100644
--- a/src/master/http.cpp
+++ b/src/master/http.cpp
@@ -30,19 +30,28 @@
#include <mesos/authorizer/authorizer.hpp>
+#include <mesos/maintenance/maintenance.hpp>
+
+#include <process/defer.hpp>
#include <process/help.hpp>
#include <process/metrics/metrics.hpp>
#include <stout/base64.hpp>
#include <stout/foreach.hpp>
+#include <stout/hashmap.hpp>
+#include <stout/hashset.hpp>
#include <stout/json.hpp>
#include <stout/lambda.hpp>
#include <stout/net.hpp>
+#include <stout/nothing.hpp>
#include <stout/numify.hpp>
#include <stout/os.hpp>
+#include <stout/protobuf.hpp>
#include <stout/result.hpp>
#include <stout/strings.hpp>
+#include <stout/try.hpp>
+#include <stout/utils.hpp>
#include "common/attributes.hpp"
#include "common/build.hpp"
@@ -53,6 +62,7 @@
#include "logging/logging.hpp"
+#include "master/maintenance.hpp"
#include "master/master.hpp"
#include "master/validation.hpp"
@@ -98,6 +108,7 @@ using mesos::internal::model;
// Pull in definitions from process.
using process::http::Response;
using process::http::Request;
+using process::Owned;
// TODO(bmahler): Kill these in favor of automatic Proto->JSON Conversion (when
@@ -1349,6 +1360,121 @@ Future<Response> Master::Http::tasks(const Request& request) const
}
+// /master/maintenance/schedule endpoint help.
+const string Master::Http::MAINTENANCE_SCHEDULE_HELP = HELP(
+ TLDR(
+ "Returns or updates the cluster's maintenance schedule."),
+ USAGE(
+ "/master/maintenance/schedule"),
+ DESCRIPTION(
+ "GET: Returns the current maintenance schedule as JSON.",
+ "POST: Validates the request body as JSON",
+ " and updates the maintenance schedule."));
+
+
+// /master/maintenance/schedule endpoint handler.
+Future<Response> Master::Http::maintenanceSchedule(const Request& request) const
+{
+ if (request.method != "GET" && request.method != "POST") {
+ return BadRequest("Expecting GET or POST, got '" + request.method + "'");
+ }
+
+ // JSON-ify and return the current maintenance schedule.
+ if (request.method == "GET") {
+ // TODO(josephw): Return more than one schedule.
+ const mesos::maintenance::Schedule schedule =
+ master->maintenance.schedules.empty() ?
+ mesos::maintenance::Schedule() :
+ master->maintenance.schedules.front();
+
+ return OK(JSON::Protobuf(schedule), request.query.get("jsonp"));
+ }
+
+ // Parse the POST body as JSON.
+ Try<JSON::Object> jsonSchedule = JSON::parse<JSON::Object>(request.body);
+ if (jsonSchedule.isError()) {
+ return BadRequest(jsonSchedule.error());
+ }
+
+ // Convert the schedule to a protobuf.
+ Try<mesos::maintenance::Schedule> protoSchedule =
+ ::protobuf::parse<mesos::maintenance::Schedule>(jsonSchedule.get());
+
+ if (protoSchedule.isError()) {
+ return BadRequest(protoSchedule.error());
+ }
+
+ // Validate that the schedule only transitions machines between
+ // `UP` and `DRAINING` modes.
+ mesos::maintenance::Schedule schedule = protoSchedule.get();
+ Try<Nothing> isValid = maintenance::validation::schedule(
+ schedule,
+ master->machineInfos);
+
+ if (isValid.isError()) {
+ return BadRequest(isValid.error());
+ }
+
+ return master->registrar->apply(Owned<Operation>(
+ new maintenance::UpdateSchedule(schedule)))
+ .then(defer(master->self(), [=](bool result) -> Future<Response> {
+ // See the top comment in "master/maintenance.hpp" for why this check
+ // is here, and is appropriate.
+ CHECK(result);
+
+ // Update the master's local state with the new schedule.
+ // NOTE: We only add or remove differences between the current schedule
+ // and the new schedule. This is because the `MachineInfo` struct
+ // holds more information than a maintenance schedule.
+ // For example, the `mode` field is not part of a maintenance schedule.
+
+ // TODO(josephw): allow more than one schedule.
+
+ // Put the machines in the updated schedule into a set.
+ // Save the unavailability, to help with updating some machines.
+ hashmap<MachineID, Unavailability> updated;
+ foreach (const mesos::maintenance::Window& window, schedule.windows()) {
+ foreach (const MachineID& id, window.machine_ids()) {
+ updated[id] = window.unavailability();
+ }
+ }
+
+ // NOTE: Copies are needed because this loop modifies the container.
+ foreachkey (const MachineID& id, utils::copy(master->machineInfos)) {
+ // Update the entry for each updated machine.
+ if (updated.contains(id)) {
+ master->machineInfos[id]
+ .mutable_unavailability()->CopyFrom(updated[id]);
+
+ continue;
+ }
+
+ // Delete the entry for each removed machine.
+ master->machineInfos.erase(id);
+ }
+
+ // Save each new machine, with the unavailability
+ // and starting in `DRAINING` mode.
+ foreach (const mesos::maintenance::Window& window, schedule.windows()) {
+ foreach (const MachineID& id, window.machine_ids()) {
+ MachineInfo info;
+ info.mutable_id()->CopyFrom(id);
+ info.set_mode(MachineInfo::DRAINING);
+ info.mutable_unavailability()->CopyFrom(window.unavailability());
+
+ master->machineInfos[id] = info;
+ }
+ }
+
+ // Replace the old schedule(s) with the new schedule.
+ master->maintenance.schedules.clear();
+ master->maintenance.schedules.push_back(schedule);
+
+ return OK();
+ }));
+}
+
+
Result<Credential> Master::Http::authenticate(const Request& request) const
{
// By default, assume everyone is authenticated if no credentials
http://git-wip-us.apache.org/repos/asf/mesos/blob/3b0fe5cc/src/master/maintenance.cpp
----------------------------------------------------------------------
diff --git a/src/master/maintenance.cpp b/src/master/maintenance.cpp
index 221dd02..798c026 100644
--- a/src/master/maintenance.cpp
+++ b/src/master/maintenance.cpp
@@ -22,6 +22,7 @@
#include <stout/duration.hpp>
#include <stout/error.hpp>
+#include <stout/hashmap.hpp>
#include <stout/hashset.hpp>
#include <stout/ip.hpp>
#include <stout/nothing.hpp>
@@ -36,6 +37,81 @@ namespace maintenance {
using namespace mesos::maintenance;
+UpdateSchedule::UpdateSchedule(
+ const maintenance::Schedule& _schedule)
+ : schedule(_schedule) {}
+
+
+Try<bool> UpdateSchedule::perform(
+ Registry* registry,
+ hashset<SlaveID>* slaveIDs,
+ bool strict)
+{
+ // Put the machines in the existing schedule into a set.
+ hashset<MachineID> existing;
+ foreach (const maintenance::Schedule& agenda, registry->schedules()) {
+ foreach (const maintenance::Window& window, agenda.windows()) {
+ foreach (const MachineID& id, window.machine_ids()) {
+ existing.insert(id);
+ }
+ }
+ }
+
+ // Put the machines in the updated schedule into a set.
+ // Keep the relevant unavailability to help update existing machines.
+ hashmap<MachineID, Unavailability> updated;
+ foreach (const maintenance::Window& window, schedule.windows()) {
+ foreach (const MachineID& id, window.machine_ids()) {
+ updated[id] = window.unavailability();
+ }
+ }
+
+ // This operation overwrites the existing schedules with a new one.
+ // At the same time, every machine in the schedule is saved as a
+ // `MachineInfo` in the registry.
+
+ // TODO(josephw): allow more than one schedule.
+
+ // Loop through and modify the existing `MachineInfo` entries.
+ for (int i = registry->machines().machines().size() - 1; i >= 0; i--) {
+ const MachineID& id = registry->machines().machines(i).info().id();
+
+ // Update the `MachineInfo` entry for all machines in the schedule.
+ if (updated.contains(id)) {
+ registry->mutable_machines()->mutable_machines(i)->mutable_info()
+ ->mutable_unavailability()->CopyFrom(updated[id]);
+
+ continue;
+ }
+
+ // Delete the `MachineInfo` entry for each removed machine.
+ registry->mutable_machines()->mutable_machines()->DeleteSubrange(i, 1);
+ }
+
+ // Create new `MachineInfo` entries for each new machine.
+ foreach (const maintenance::Window& window, schedule.windows()) {
+ foreach (const MachineID& id, window.machine_ids()) {
+ if (existing.contains(id)) {
+ continue;
+ }
+
+ // Each newly scheduled machine starts in `DRAINING` mode.
+ Registry::Machine* machine = registry->mutable_machines()->add_machines();
+ MachineInfo* info = machine->mutable_info();
+ info->mutable_id()->CopyFrom(id);
+ info->set_mode(MachineInfo::DRAINING);
+ info->mutable_unavailability()->CopyFrom(window.unavailability());
+ }
+ }
+
+ // Replace the old schedule with the new schedule.
+ registry->clear_schedules();
+ registry->add_schedules()->CopyFrom(schedule);
+
+ return true; // Mutation.
+}
+
+
namespace validation {
Try<Nothing> schedule(
http://git-wip-us.apache.org/repos/asf/mesos/blob/3b0fe5cc/src/master/maintenance.hpp
----------------------------------------------------------------------
diff --git a/src/master/maintenance.hpp b/src/master/maintenance.hpp
index 833665e..42b5f9e 100644
--- a/src/master/maintenance.hpp
+++ b/src/master/maintenance.hpp
@@ -34,6 +34,37 @@ namespace mesos {
namespace internal {
namespace master {
namespace maintenance {
+
+// Maintenance registry operations will never report a failure while
+// performing the operation (i.e. `perform` never returns an `Error`).
+// This means the underlying `Future` for the operation will always be
+// set to true. If there is a separate failure, such as a network
+// partition, while performing the operation, this future will not be set.
+
+/**
+ * Updates the maintanence schedule of the cluster. This transitions machines
+ * between `UP` and `DRAINING` modes only. The given schedule must only
+ * add valid machines and remove machines that are not `DOWN`.
+ *
+ * TODO(josephw): allow more than one schedule.
+ */
+class UpdateSchedule : public Operation
+{
+public:
+ explicit UpdateSchedule(
+ const mesos::maintenance::Schedule& _schedule);
+
+protected:
+ Try<bool> perform(
+ Registry* registry,
+ hashset<SlaveID>* slaveIDs,
+ bool strict);
+
+private:
+ const mesos::maintenance::Schedule schedule;
+};
+
+
namespace validation {
/**
http://git-wip-us.apache.org/repos/asf/mesos/blob/3b0fe5cc/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 564fbcb..ea556f9 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -815,6 +815,12 @@ void Master::initialize()
Http::log(request);
return http.tasks(request);
});
+ route("/maintenance/schedule",
+ Http::MAINTENANCE_SCHEDULE_HELP,
+ [http](const process::http::Request& request) {
+ Http::log(request);
+ return http.maintenanceSchedule(request);
+ });
// Provide HTTP assets from a "webui" directory. This is either
// specified via flags (which is necessary for running out of the
http://git-wip-us.apache.org/repos/asf/mesos/blob/3b0fe5cc/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index edcbeed..175e623 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -832,6 +832,10 @@ private:
process::Future<process::http::Response> tasks(
const process::http::Request& request) const;
+ // /master/maintenance/schedule
+ process::Future<process::http::Response> maintenanceSchedule(
+ const process::http::Request& request) const;
+
const static std::string SCHEDULER_HELP;
const static std::string HEALTH_HELP;
const static std::string OBSERVE_HELP;
@@ -842,6 +846,7 @@ private:
const static std::string STATE_HELP;
const static std::string STATESUMMARY_HELP;
const static std::string TASKS_HELP;
+ const static std::string MAINTENANCE_SCHEDULE_HELP;
private:
// Helper for doing authentication, returns the credential used if
http://git-wip-us.apache.org/repos/asf/mesos/blob/3b0fe5cc/src/tests/master_maintenance_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/master_maintenance_tests.cpp b/src/tests/master_maintenance_tests.cpp
new file mode 100644
index 0000000..1258ecc
--- /dev/null
+++ b/src/tests/master_maintenance_tests.cpp
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string>
+
+#include <process/clock.hpp>
+#include <process/future.hpp>
+#include <process/http.hpp>
+#include <process/pid.hpp>
+#include <process/time.hpp>
+
+#include <stout/duration.hpp>
+#include <stout/json.hpp>
+#include <stout/net.hpp>
+#include <stout/option.hpp>
+#include <stout/strings.hpp>
+#include <stout/stringify.hpp>
+#include <stout/try.hpp>
+
+#include "common/protobuf_utils.hpp"
+
+#include "master/master.hpp"
+
+#include "slave/flags.hpp"
+
+#include "tests/mesos.hpp"
+#include "tests/utils.hpp"
+
+using mesos::internal::master::Master;
+
+using mesos::internal::slave::Slave;
+
+using process::Clock;
+using process::Future;
+using process::PID;
+using process::Time;
+
+using process::http::BadRequest;
+using process::http::OK;
+using process::http::Response;
+
+using mesos::internal::protobuf::maintenance::createSchedule;
+using mesos::internal::protobuf::maintenance::createUnavailability;
+using mesos::internal::protobuf::maintenance::createWindow;
+
+using std::string;
+
+using testing::DoAll;
+
+namespace mesos {
+namespace internal {
+namespace tests {
+
+class MasterMaintenanceTest : public MesosTest
+{
+public:
+ virtual void SetUp()
+ {
+ MesosTest::SetUp();
+
+ // Initialize the default POST header.
+ headers["Content-Type"] = "application/json";
+
+ // Initialize some `MachineID`s.
+ machine1.set_hostname("Machine1");
+ machine2.set_ip("0.0.0.2");
+ machine3.set_hostname("Machine3");
+ machine3.set_ip("0.0.0.3");
+
+ // Initialize the default `Unavailability`.
+ unavailability = createUnavailability(Clock::now());
+ }
+
+
+ // Default headers for all POST's to maintenance endpoints.
+ hashmap<string, string> headers;
+
+ // Some generic `MachineID`s that can be used in this test.
+ MachineID machine1;
+ MachineID machine2;
+ MachineID machine3;
+
+ // Default unavailability. Used when the test does not care
+ // about the value of the unavailability.
+ Unavailability unavailability;
+};
+
+
+// Posts valid and invalid schedules to the maintenance schedule endpoint.
+TEST_F(MasterMaintenanceTest, UpdateSchedule)
+{
+ // Set up a master.
+ Try<PID<Master>> master = StartMaster();
+ ASSERT_SOME(master);
+
+ // Extra machine used in this test.
+ // It isn't filled in, so it's incorrect.
+ MachineID badMachine;
+
+ // Post a valid schedule with one machine.
+ maintenance::Schedule schedule = createSchedule(
+ {createWindow({machine1}, unavailability)});
+
+ Future<Response> response = process::http::post(
+ master.get(),
+ "maintenance/schedule",
+ headers,
+ stringify(JSON::Protobuf(schedule)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+
+ // Get the maintenance schedule.
+ response = process::http::get(
+ master.get(),
+ "maintenance/schedule");
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+
+ // Check that the schedule was saved.
+ Try<JSON::Object> masterSchedule_ =
+ JSON::parse<JSON::Object>(response.get().body);
+
+ ASSERT_SOME(masterSchedule_);
+ Try<mesos::maintenance::Schedule> masterSchedule =
+ ::protobuf::parse<mesos::maintenance::Schedule>(masterSchedule_.get());
+
+ ASSERT_SOME(masterSchedule);
+ ASSERT_EQ(1, masterSchedule.get().windows().size());
+ ASSERT_EQ(1, masterSchedule.get().windows(0).machine_ids().size());
+ ASSERT_EQ(
+ "Machine1",
+ masterSchedule.get().windows(0).machine_ids(0).hostname());
+
+ // Try to replace with an invalid schedule with an empty window.
+ schedule = createSchedule(
+ {createWindow({}, unavailability)});
+
+ response = process::http::post(
+ master.get(),
+ "maintenance/schedule",
+ headers,
+ stringify(JSON::Protobuf(schedule)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(BadRequest().status, response);
+
+ // Update the schedule with an unavailability with negative time.
+ schedule = createSchedule({
+ createWindow(
+ {machine1},
+ createUnavailability(Time::create(-10).get()))});
+
+ response = process::http::post(
+ master.get(),
+ "maintenance/schedule",
+ headers,
+ stringify(JSON::Protobuf(schedule)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+
+ // Try to replace with an invalid schedule with a negative duration.
+ schedule = createSchedule({
+ createWindow(
+ {machine1},
+ createUnavailability(Clock::now(), Seconds(-10)))});
+
+ response = process::http::post(
+ master.get(),
+ "maintenance/schedule",
+ headers,
+ stringify(JSON::Protobuf(schedule)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(BadRequest().status, response);
+
+ // Try to replace with another invalid schedule with duplicate machines.
+ schedule = createSchedule({
+ createWindow({machine1}, unavailability),
+ createWindow({machine1}, unavailability)});
+
+ response = process::http::post(
+ master.get(),
+ "maintenance/schedule",
+ headers,
+ stringify(JSON::Protobuf(schedule)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(BadRequest().status, response);
+
+ // Try to replace with an invalid schedule with a badly formed MachineInfo.
+ schedule = createSchedule(
+ {createWindow({badMachine}, unavailability)});
+
+ response = process::http::post(
+ master.get(),
+ "maintenance/schedule",
+ headers,
+ stringify(JSON::Protobuf(schedule)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(BadRequest().status, response);
+
+ // Post a valid schedule with two machines.
+ schedule = createSchedule(
+ {createWindow({machine1, machine2}, unavailability)});
+
+ response = process::http::post(
+ master.get(),
+ "maintenance/schedule",
+ headers,
+ stringify(JSON::Protobuf(schedule)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+
+ // Delete the schedule (via an empty schedule).
+ schedule = createSchedule({});
+ response = process::http::post(
+ master.get(),
+ "maintenance/schedule",
+ headers,
+ stringify(JSON::Protobuf(schedule)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+}
+
+} // namespace tests {
+} // namespace internal {
+} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/3b0fe5cc/src/tests/registrar_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/registrar_tests.cpp b/src/tests/registrar_tests.cpp
index 032e644..567934c 100644
--- a/src/tests/registrar_tests.cpp
+++ b/src/tests/registrar_tests.cpp
@@ -25,6 +25,7 @@
#include <mesos/type_utils.hpp>
+#include <process/clock.hpp>
#include <process/gmock.hpp>
#include <process/gtest.hpp>
#include <process/pid.hpp>
@@ -45,6 +46,7 @@
#include "messages/state.hpp"
#include "master/flags.hpp"
+#include "master/maintenance.hpp"
#include "master/master.hpp"
#include "master/registrar.hpp"
@@ -68,6 +70,12 @@ using std::set;
using std::string;
using std::vector;
+using process::Clock;
+
+using mesos::internal::protobuf::maintenance::createSchedule;
+using mesos::internal::protobuf::maintenance::createUnavailability;
+using mesos::internal::protobuf::maintenance::createWindow;
+
using testing::_;
using testing::DoAll;
using testing::Eq;
@@ -81,6 +89,9 @@ namespace mesos {
namespace internal {
namespace tests {
+using namespace mesos::maintenance;
+using namespace mesos::internal::master::maintenance;
+
using state::Entry;
using state::LogStorage;
using state::Storage;
@@ -316,6 +327,179 @@ TEST_P(RegistrarTest, Remove)
}
+// NOTE: For the following tests, the state of the registrar can
+// only be viewed once per instantiation of the registrar.
+// To check the result of each operation, we must re-construct
+// the registrar, which is done by putting the code into scoped blocks.
+
+// TODO(josephw): Consider refactoring these maintenance operation tests
+// to use a helper function for each un-named scoped block.
+// For example:
+// MaintenanceTest(flags, state, [=](const Registry& registry) {
+// // Checks and operations. i.e.:
+// EXPECT_EQ(1, registry.get().schedules().size());
+// });
+
+// Adds maintenance schedules to the registry, one machine at a time.
+// Then removes machines from the schedule.
+TEST_P(RegistrarTest, UpdateMaintenanceSchedule)
+{
+ // Machine definitions used in this test.
+ MachineID machine1;
+ machine1.set_ip("0.0.0.1");
+
+ MachineID machine2;
+ machine2.set_hostname("2");
+
+ MachineID machine3;
+ machine3.set_hostname("3");
+ machine3.set_ip("0.0.0.3");
+
+ Unavailability unavailability = createUnavailability(Clock::now());
+
+ {
+ // Prepare the registrar.
+ Registrar registrar(flags, state);
+ AWAIT_READY(registrar.recover(master));
+
+ // Schedule one machine for maintenance.
+ maintenance::Schedule schedule = createSchedule(
+ {createWindow({machine1}, unavailability)});
+
+ AWAIT_READY(registrar.apply(
+ Owned<Operation>(new UpdateSchedule(schedule))));
+ }
+
+ {
+ // Check that one schedule and one machine info was made.
+ Registrar registrar(flags, state);
+ Future<Registry> registry = registrar.recover(master);
+ AWAIT_READY(registry);
+
+ EXPECT_EQ(1, registry.get().schedules().size());
+ EXPECT_EQ(1, registry.get().schedules(0).windows().size());
+ EXPECT_EQ(1, registry.get().schedules(0).windows(0).machine_ids().size());
+ EXPECT_EQ(1, registry.get().machines().machines().size());
+ EXPECT_EQ(
+ MachineInfo::DRAINING,
+ registry.get().machines().machines(0).info().mode());
+
+ // Extend the schedule by one machine (in a different window).
+ maintenance::Schedule schedule = createSchedule({
+ createWindow({machine1}, unavailability),
+ createWindow({machine2}, unavailability)});
+
+ AWAIT_READY(registrar.apply(
+ Owned<Operation>(new UpdateSchedule(schedule))));
+ }
+
+ {
+ // Check that both machines are part of maintenance.
+ Registrar registrar(flags, state);
+ Future<Registry> registry = registrar.recover(master);
+ AWAIT_READY(registry);
+
+ EXPECT_EQ(1, registry.get().schedules().size());
+ EXPECT_EQ(2, registry.get().schedules(0).windows().size());
+ EXPECT_EQ(1, registry.get().schedules(0).windows(0).machine_ids().size());
+ EXPECT_EQ(1, registry.get().schedules(0).windows(1).machine_ids().size());
+ EXPECT_EQ(2, registry.get().machines().machines().size());
+ EXPECT_EQ(
+ MachineInfo::DRAINING,
+ registry.get().machines().machines(1).info().mode());
+
+ // Extend a window by one machine.
+ maintenance::Schedule schedule = createSchedule({
+ createWindow({machine1}, unavailability),
+ createWindow({machine2, machine3}, unavailability)});
+
+ AWAIT_READY(registrar.apply(
+ Owned<Operation>(new UpdateSchedule(schedule))));
+ }
+
+ {
+ // Check that all three machines are included.
+ Registrar registrar(flags, state);
+ Future<Registry> registry = registrar.recover(master);
+ AWAIT_READY(registry);
+
+ EXPECT_EQ(1, registry.get().schedules().size());
+ EXPECT_EQ(2, registry.get().schedules(0).windows().size());
+ EXPECT_EQ(1, registry.get().schedules(0).windows(0).machine_ids().size());
+ EXPECT_EQ(2, registry.get().schedules(0).windows(1).machine_ids().size());
+ EXPECT_EQ(3, registry.get().machines().machines().size());
+ EXPECT_EQ(
+ MachineInfo::DRAINING,
+ registry.get().machines().machines(2).info().mode());
+
+ // Rearrange the schedule into one window.
+ maintenance::Schedule schedule = createSchedule(
+ {createWindow({machine1, machine2, machine3}, unavailability)});
+
+ AWAIT_READY(registrar.apply(
+ Owned<Operation>(new UpdateSchedule(schedule))));
+ }
+
+ {
+ // Check that the machine infos are unchanged, but the schedule is.
+ Registrar registrar(flags, state);
+ Future<Registry> registry = registrar.recover(master);
+ AWAIT_READY(registry);
+
+ EXPECT_EQ(1, registry.get().schedules().size());
+ EXPECT_EQ(1, registry.get().schedules(0).windows().size());
+ EXPECT_EQ(3, registry.get().schedules(0).windows(0).machine_ids().size());
+ EXPECT_EQ(3, registry.get().machines().machines().size());
+ EXPECT_EQ(
+ MachineInfo::DRAINING,
+ registry.get().machines().machines(0).info().mode());
+
+ EXPECT_EQ(
+ MachineInfo::DRAINING,
+ registry.get().machines().machines(1).info().mode());
+
+ EXPECT_EQ(
+ MachineInfo::DRAINING,
+ registry.get().machines().machines(2).info().mode());
+
+ // Delete one machine from the schedule.
+ maintenance::Schedule schedule = createSchedule(
+ {createWindow({machine2, machine3}, unavailability)});
+
+ AWAIT_READY(registrar.apply(
+ Owned<Operation>(new UpdateSchedule(schedule))));
+ }
+
+ {
+ // Check that one machine info is removed.
+ Registrar registrar(flags, state);
+ Future<Registry> registry = registrar.recover(master);
+ AWAIT_READY(registry);
+
+ EXPECT_EQ(1, registry.get().schedules().size());
+ EXPECT_EQ(1, registry.get().schedules(0).windows().size());
+ EXPECT_EQ(2, registry.get().schedules(0).windows(0).machine_ids().size());
+ EXPECT_EQ(2, registry.get().machines().machines().size());
+
+ // Delete all machines from the schedule.
+ maintenance::Schedule schedule;
+ AWAIT_READY(registrar.apply(
+ Owned<Operation>(new UpdateSchedule(schedule))));
+ }
+
+ {
+ // Check that all statuses are removed.
+ Registrar registrar(flags, state);
+ Future<Registry> registry = registrar.recover(master);
+ AWAIT_READY(registry);
+
+ EXPECT_EQ(1, registry.get().schedules().size());
+ EXPECT_EQ(0, registry.get().schedules(0).windows().size());
+ EXPECT_EQ(0, registry.get().machines().machines().size());
+ }
+}
+
+
TEST_P(RegistrarTest, Bootstrap)
{
// Run 1 readmits a slave that is not present.
[6/8] mesos git commit: Maintenance Primitives: Added machine DOWN
endpoint.
Posted by jo...@apache.org.
Maintenance Primitives: Added machine DOWN endpoint.
Endpoint: /machine/down
Transitions agents into the DOWN mode.
Registry operation = maintenance::StartMaintenance
Sets the list of machines as DOWN.
Review: https://reviews.apache.org/r/37358
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/de231ed5
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/de231ed5
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/de231ed5
Branch: refs/heads/master
Commit: de231ed590809d85c9e47071f5ce899035a15dc4
Parents: 3b0fe5c
Author: Joseph Wu <jo...@mesosphere.io>
Authored: Sun Aug 30 13:56:21 2015 -0400
Committer: Joris Van Remoortere <jo...@gmail.com>
Committed: Mon Aug 31 13:09:54 2015 -0400
----------------------------------------------------------------------
src/master/http.cpp | 73 ++++++++++++++++++++++
src/master/maintenance.cpp | 30 +++++++++
src/master/maintenance.hpp | 24 ++++++++
src/master/master.cpp | 6 ++
src/master/master.hpp | 5 ++
src/tests/master_maintenance_tests.cpp | 95 +++++++++++++++++++++++++++++
src/tests/registrar_tests.cpp | 87 +++++++++++++++++++++++++-
7 files changed, 319 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/de231ed5/src/master/http.cpp
----------------------------------------------------------------------
diff --git a/src/master/http.cpp b/src/master/http.cpp
index 44178d8..11e786d 100644
--- a/src/master/http.cpp
+++ b/src/master/http.cpp
@@ -1475,6 +1475,79 @@ Future<Response> Master::Http::maintenanceSchedule(const Request& request) const
}
+// /master/machine/down endpoint help.
+const string Master::Http::MACHINE_DOWN_HELP = HELP(
+ TLDR(
+ "Brings a set of machines down."),
+ USAGE(
+ "/master/machine/down"),
+ DESCRIPTION(
+ "POST: Validates the request body as JSON and transitions",
+ " the list of machines into DOWN mode. Currently, only",
+ " machines in DRAINING mode are allowed to be brought down."));
+
+
+// /master/machine/down endpoint handler.
+Future<Response> Master::Http::machineDown(const Request& request) const
+{
+ if (request.method != "POST") {
+ return BadRequest("Expecting POST, got '" + request.method + "'");
+ }
+
+ // Parse the POST body as JSON.
+ Try<JSON::Object> jsonIds = JSON::parse<JSON::Object>(request.body);
+ if (jsonIds.isError()) {
+ return BadRequest(jsonIds.error());
+ }
+
+ // Convert the machines to a protobuf.
+ Try<MachineIDs> protoIds =
+ ::protobuf::parse<MachineIDs>(jsonIds.get());
+
+ if (protoIds.isError()) {
+ return BadRequest(protoIds.error());
+ }
+
+ // Validate every machine in the list.
+ MachineIDs ids = protoIds.get();
+ Try<Nothing> isValid = maintenance::validation::machines(ids);
+ if (isValid.isError()) {
+ return BadRequest(isValid.error());
+ }
+
+ // Check that all machines are part of a maintenance schedule.
+ // TODO(josephw): Allow a transition from `UP` to `DOWN`.
+ foreach (const MachineID& id, ids.values()) {
+ if (!master->machineInfos.contains(id)) {
+ return BadRequest(
+ "Machine '" + id.DebugString() +
+ "' is not part of a maintenance schedule");
+ }
+
+ if (master->machineInfos[id].mode() != MachineInfo::DRAINING) {
+ return BadRequest(
+ "Machine '" + id.DebugString() +
+ "' is not in DRAINING mode and cannot be brought down");
+ }
+ }
+
+ return master->registrar->apply(Owned<Operation>(
+ new maintenance::StartMaintenance(ids)))
+ .then(defer(master->self(), [=](bool result) -> Future<Response> {
+ // See the top comment in "master/maintenance.hpp" for why this check
+ // is here, and is appropriate.
+ CHECK(result);
+
+ // Update the master's local state with the downed machines.
+ foreach (const MachineID& id, ids.values()) {
+ master->machineInfos[id].set_mode(MachineInfo::DOWN);
+ }
+
+ return OK();
+ }));
+}
+
+
Result<Credential> Master::Http::authenticate(const Request& request) const
{
// By default, assume everyone is authenticated if no credentials
http://git-wip-us.apache.org/repos/asf/mesos/blob/de231ed5/src/master/maintenance.cpp
----------------------------------------------------------------------
diff --git a/src/master/maintenance.cpp b/src/master/maintenance.cpp
index 798c026..859cef9 100644
--- a/src/master/maintenance.cpp
+++ b/src/master/maintenance.cpp
@@ -112,6 +112,36 @@ Try<bool> UpdateSchedule::perform(
}
+StartMaintenance::StartMaintenance(
+ const MachineIDs& _ids)
+{
+ foreach (const MachineID& id, _ids.values()) {
+ ids.insert(id);
+ }
+}
+
+
+Try<bool> StartMaintenance::perform(
+ Registry* registry,
+ hashset<SlaveID>* slaveIDs,
+ bool strict)
+{
+ // Flip the mode of all targeted machines.
+ bool changed = false;
+ for (int i = 0; i < registry->machines().machines().size(); i++) {
+ if (ids.contains(registry->machines().machines(i).info().id())) {
+ // Flip the mode.
+ registry->mutable_machines()->mutable_machines(i)
+ ->mutable_info()->set_mode(MachineInfo::DOWN);
+
+ changed = true; // Mutation.
+ }
+ }
+
+ return changed;
+}
+
+
namespace validation {
Try<Nothing> schedule(
http://git-wip-us.apache.org/repos/asf/mesos/blob/de231ed5/src/master/maintenance.hpp
----------------------------------------------------------------------
diff --git a/src/master/maintenance.hpp b/src/master/maintenance.hpp
index 42b5f9e..8e6cb9c 100644
--- a/src/master/maintenance.hpp
+++ b/src/master/maintenance.hpp
@@ -65,6 +65,30 @@ private:
};
+/**
+ * Transitions a group of machines from `DRAINING` mode into
+ * `DOWN` mode. All machines must be part of a maintenance
+ * schedule prior to executing this operation.
+ *
+ * TODO(josephw): Allow a transition from `UP` to `DOWN`.
+ */
+class StartMaintenance : public Operation
+{
+public:
+ explicit StartMaintenance(
+ const MachineIDs& _ids);
+
+protected:
+ Try<bool> perform(
+ Registry* registry,
+ hashset<SlaveID>* slaveIDs,
+ bool strict);
+
+private:
+ hashset<MachineID> ids;
+};
+
+
namespace validation {
/**
http://git-wip-us.apache.org/repos/asf/mesos/blob/de231ed5/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index ea556f9..06e283d 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -821,6 +821,12 @@ void Master::initialize()
Http::log(request);
return http.maintenanceSchedule(request);
});
+ route("/machine/down",
+ Http::MACHINE_DOWN_HELP,
+ [http](const process::http::Request& request) {
+ Http::log(request);
+ return http.machineDown(request);
+ });
// Provide HTTP assets from a "webui" directory. This is either
// specified via flags (which is necessary for running out of the
http://git-wip-us.apache.org/repos/asf/mesos/blob/de231ed5/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index 175e623..c7e96db 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -836,6 +836,10 @@ private:
process::Future<process::http::Response> maintenanceSchedule(
const process::http::Request& request) const;
+ // /master/machine/down
+ process::Future<process::http::Response> machineDown(
+ const process::http::Request& request) const;
+
const static std::string SCHEDULER_HELP;
const static std::string HEALTH_HELP;
const static std::string OBSERVE_HELP;
@@ -847,6 +851,7 @@ private:
const static std::string STATESUMMARY_HELP;
const static std::string TASKS_HELP;
const static std::string MAINTENANCE_SCHEDULE_HELP;
+ const static std::string MACHINE_DOWN_HELP;
private:
// Helper for doing authentication, returns the credential used if
http://git-wip-us.apache.org/repos/asf/mesos/blob/de231ed5/src/tests/master_maintenance_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/master_maintenance_tests.cpp b/src/tests/master_maintenance_tests.cpp
index 1258ecc..67301cc 100644
--- a/src/tests/master_maintenance_tests.cpp
+++ b/src/tests/master_maintenance_tests.cpp
@@ -54,6 +54,7 @@ using process::http::BadRequest;
using process::http::OK;
using process::http::Response;
+using mesos::internal::protobuf::maintenance::createMachineList;
using mesos::internal::protobuf::maintenance::createSchedule;
using mesos::internal::protobuf::maintenance::createUnavailability;
using mesos::internal::protobuf::maintenance::createWindow;
@@ -234,6 +235,100 @@ TEST_F(MasterMaintenanceTest, UpdateSchedule)
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
}
+
+// Posts valid and invalid machines to the maintenance start endpoint.
+TEST_F(MasterMaintenanceTest, BringDownMachines)
+{
+ // Set up a master.
+ Try<PID<Master>> master = StartMaster();
+ ASSERT_SOME(master);
+
+ // Extra machine used in this test.
+ // It isn't filled in, so it's incorrect.
+ MachineID badMachine;
+
+ // Try to start maintenance on an unscheduled machine.
+ MachineIDs machines = createMachineList({machine1, machine2});
+ Future<Response> response = process::http::post(
+ master.get(),
+ "machine/down",
+ headers,
+ stringify(JSON::Protobuf(machines)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(BadRequest().status, response);
+
+ // Try an empty list.
+ machines = createMachineList({});
+ response = process::http::post(
+ master.get(),
+ "machine/down",
+ headers,
+ stringify(JSON::Protobuf(machines)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(BadRequest().status, response);
+
+ // Try an empty machine.
+ machines = createMachineList({badMachine});
+ response = process::http::post(
+ master.get(),
+ "machine/down",
+ headers,
+ stringify(JSON::Protobuf(machines)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(BadRequest().status, response);
+
+ // Post a valid schedule with two machines.
+ maintenance::Schedule schedule = createSchedule(
+ {createWindow({machine1, machine2}, unavailability)});
+
+ response = process::http::post(
+ master.get(),
+ "maintenance/schedule",
+ headers,
+ stringify(JSON::Protobuf(schedule)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+
+ // Down machine1.
+ machines = createMachineList({machine1});
+ response = process::http::post(
+ master.get(),
+ "machine/down",
+ headers,
+ stringify(JSON::Protobuf(machines)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+
+ // Fail to down machine1 again.
+ response = process::http::post(
+ master.get(),
+ "machine/down",
+ headers,
+ stringify(JSON::Protobuf(machines)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(BadRequest().status, response);
+
+ // Fail to down machine1 and machine2.
+ machines = createMachineList({machine1, machine2});
+ response = process::http::post(
+ master.get(),
+ "machine/down",
+ headers,
+ stringify(JSON::Protobuf(machines)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(BadRequest().status, response);
+
+ // Down machine2.
+ machines = createMachineList({machine2});
+ response = process::http::post(
+ master.get(),
+ "machine/down",
+ headers,
+ stringify(JSON::Protobuf(machines)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+}
+
} // namespace tests {
} // namespace internal {
} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/de231ed5/src/tests/registrar_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/registrar_tests.cpp b/src/tests/registrar_tests.cpp
index 567934c..733a2cd 100644
--- a/src/tests/registrar_tests.cpp
+++ b/src/tests/registrar_tests.cpp
@@ -72,6 +72,7 @@ using std::vector;
using process::Clock;
+using mesos::internal::protobuf::maintenance::createMachineList;
using mesos::internal::protobuf::maintenance::createSchedule;
using mesos::internal::protobuf::maintenance::createUnavailability;
using mesos::internal::protobuf::maintenance::createWindow;
@@ -492,7 +493,7 @@ TEST_P(RegistrarTest, UpdateMaintenanceSchedule)
Registrar registrar(flags, state);
Future<Registry> registry = registrar.recover(master);
AWAIT_READY(registry);
-
+
EXPECT_EQ(1, registry.get().schedules().size());
EXPECT_EQ(0, registry.get().schedules(0).windows().size());
EXPECT_EQ(0, registry.get().machines().machines().size());
@@ -500,6 +501,90 @@ TEST_P(RegistrarTest, UpdateMaintenanceSchedule)
}
+// Creates a schedule and properly starts maintenance.
+TEST_P(RegistrarTest, StartMaintenance)
+{
+ // Machine definitions used in this test.
+ MachineID machine1;
+ machine1.set_ip("0.0.0.1");
+
+ MachineID machine2;
+ machine2.set_hostname("2");
+
+ MachineID machine3;
+ machine3.set_hostname("3");
+ machine3.set_ip("0.0.0.3");
+
+ Unavailability unavailability = createUnavailability(Clock::now());
+
+ {
+ // Prepare the registrar.
+ Registrar registrar(flags, state);
+ AWAIT_READY(registrar.recover(master));
+
+ // Schedule two machines for maintenance.
+ maintenance::Schedule schedule = createSchedule(
+ {createWindow({machine1, machine2}, unavailability)});
+
+ AWAIT_READY(registrar.apply(
+ Owned<Operation>(new UpdateSchedule(schedule))));
+
+ // Transition machine two into `DOWN` mode.
+ MachineIDs machines = createMachineList({machine2});
+ AWAIT_READY(registrar.apply(
+ Owned<Operation>(new StartMaintenance(machines))));
+ }
+
+ {
+ // Check that machine two is down.
+ Registrar registrar(flags, state);
+ Future<Registry> registry = registrar.recover(master);
+ AWAIT_READY(registry);
+
+ EXPECT_EQ(2, registry.get().machines().machines().size());
+ EXPECT_EQ(
+ MachineInfo::DRAINING,
+ registry.get().machines().machines(0).info().mode());
+
+ EXPECT_EQ(
+ MachineInfo::DOWN,
+ registry.get().machines().machines(1).info().mode());
+
+ // Schedule three machines for maintenance.
+ maintenance::Schedule schedule = createSchedule(
+ {createWindow({machine1, machine2, machine3}, unavailability)});
+
+ AWAIT_READY(registrar.apply(
+ Owned<Operation>(new UpdateSchedule(schedule))));
+
+ // Deactivate the two `DRAINING` machines.
+ MachineIDs machines = createMachineList({machine1, machine3});
+ AWAIT_READY(registrar.apply(
+ Owned<Operation>(new StartMaintenance(machines))));
+ }
+
+ {
+ // Check that all machines are down.
+ Registrar registrar(flags, state);
+ Future<Registry> registry = registrar.recover(master);
+ AWAIT_READY(registry);
+
+ EXPECT_EQ(3, registry.get().machines().machines().size());
+ EXPECT_EQ(
+ MachineInfo::DOWN,
+ registry.get().machines().machines(0).info().mode());
+
+ EXPECT_EQ(
+ MachineInfo::DOWN,
+ registry.get().machines().machines(1).info().mode());
+
+ EXPECT_EQ(
+ MachineInfo::DOWN,
+ registry.get().machines().machines(2).info().mode());
+ }
+}
+
+
TEST_P(RegistrarTest, Bootstrap)
{
// Run 1 readmits a slave that is not present.
[8/8] mesos git commit: Maintenance Primitives: Added maintenance
status endpoint.
Posted by jo...@apache.org.
Maintenance Primitives: Added maintenance status endpoint.
Endpoint: /maintenance/status
Retrieves the maintenance status of machines.
Also adds a protobuf for the purpose of serialization.
Review: https://reviews.apache.org/r/37364
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/09e4ba2f
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/09e4ba2f
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/09e4ba2f
Branch: refs/heads/master
Commit: 09e4ba2faf8eb8f4b82b640b641ec4199a11001d
Parents: bf4ca54
Author: Joseph Wu <jo...@mesosphere.io>
Authored: Sun Aug 30 13:56:45 2015 -0400
Committer: Joris Van Remoortere <jo...@gmail.com>
Committed: Mon Aug 31 13:15:18 2015 -0400
----------------------------------------------------------------------
include/mesos/maintenance/maintenance.proto | 10 ++
src/master/http.cpp | 41 +++++++
src/master/master.cpp | 6 +
src/master/master.hpp | 5 +
src/tests/master_maintenance_tests.cpp | 139 +++++++++++++++++++++++
5 files changed, 201 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/09e4ba2f/include/mesos/maintenance/maintenance.proto
----------------------------------------------------------------------
diff --git a/include/mesos/maintenance/maintenance.proto b/include/mesos/maintenance/maintenance.proto
index ba28d0f..ee01c5d 100644
--- a/include/mesos/maintenance/maintenance.proto
+++ b/include/mesos/maintenance/maintenance.proto
@@ -65,3 +65,13 @@ message Window {
message Schedule {
repeated Window windows = 1;
}
+
+
+/**
+ * Represents the maintenance status of each machine in the cluster.
+ * Corresponds to the `MachineInfo.Mode` enumeration.
+ */
+message ClusterStatus {
+ repeated MachineID draining_machines = 1;
+ repeated MachineID down_machines = 2;
+}
http://git-wip-us.apache.org/repos/asf/mesos/blob/09e4ba2f/src/master/http.cpp
----------------------------------------------------------------------
diff --git a/src/master/http.cpp b/src/master/http.cpp
index 6fad959..94e97a2 100644
--- a/src/master/http.cpp
+++ b/src/master/http.cpp
@@ -1651,6 +1651,47 @@ Future<Response> Master::Http::machineUp(const Request& request) const
}
+// /master/maintenance/status endpoint help.
+const string Master::Http::MAINTENANCE_STATUS_HELP = HELP(
+ TLDR(
+ "Retrieves the maintenance status of the cluster."),
+ USAGE(
+ "/master/maintenance/status"),
+ DESCRIPTION(
+ "Returns an object with one list of machines per machine mode."));
+
+
+// /master/maintenance/status endpoint handler.
+Future<Response> Master::Http::maintenanceStatus(const Request& request) const
+{
+ if (request.method != "GET") {
+ return BadRequest("Expecting GET, got '" + request.method + "'");
+ }
+
+ // Unwrap the master's machine information into two arrays of machines.
+ mesos::maintenance::ClusterStatus status;
+ foreachkey (const MachineID& id, master->machineInfos) {
+ switch (master->machineInfos[id].mode()) {
+ case MachineInfo::DRAINING: {
+ status.add_draining_machines()->CopyFrom(id);
+ break;
+ }
+ case MachineInfo::DOWN: {
+ status.add_down_machines()->CopyFrom(id);
+ break;
+ }
+ // Currently, `UP` machines are not specifically tracked in the master.
+ case MachineInfo::UP: {}
+ default: {
+ break;
+ }
+ }
+ }
+
+ return OK(JSON::Protobuf(status), request.query.get("jsonp"));
+}
+
+
Result<Credential> Master::Http::authenticate(const Request& request) const
{
// By default, assume everyone is authenticated if no credentials
http://git-wip-us.apache.org/repos/asf/mesos/blob/09e4ba2f/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index cd1b386..56bcbcc 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -821,6 +821,12 @@ void Master::initialize()
Http::log(request);
return http.maintenanceSchedule(request);
});
+ route("/maintenance/status",
+ Http::MAINTENANCE_STATUS_HELP,
+ [http](const process::http::Request& request) {
+ Http::log(request);
+ return http.maintenanceStatus(request);
+ });
route("/machine/down",
Http::MACHINE_DOWN_HELP,
[http](const process::http::Request& request) {
http://git-wip-us.apache.org/repos/asf/mesos/blob/09e4ba2f/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index 68be718..594dd25 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -836,6 +836,10 @@ private:
process::Future<process::http::Response> maintenanceSchedule(
const process::http::Request& request) const;
+ // /master/maintenance/status
+ process::Future<process::http::Response> maintenanceStatus(
+ const process::http::Request& request) const;
+
// /master/machine/down
process::Future<process::http::Response> machineDown(
const process::http::Request& request) const;
@@ -855,6 +859,7 @@ private:
const static std::string STATESUMMARY_HELP;
const static std::string TASKS_HELP;
const static std::string MAINTENANCE_SCHEDULE_HELP;
+ const static std::string MAINTENANCE_STATUS_HELP;
const static std::string MACHINE_DOWN_HELP;
const static std::string MACHINE_UP_HELP;
http://git-wip-us.apache.org/repos/asf/mesos/blob/09e4ba2f/src/tests/master_maintenance_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/master_maintenance_tests.cpp b/src/tests/master_maintenance_tests.cpp
index c3acd7a..fb8dca3 100644
--- a/src/tests/master_maintenance_tests.cpp
+++ b/src/tests/master_maintenance_tests.cpp
@@ -239,6 +239,58 @@ TEST_F(MasterMaintenanceTest, UpdateSchedule)
}
+// Tries to remove deactivated machines from the schedule.
+TEST_F(MasterMaintenanceTest, FailToUnscheduleDeactivatedMachines)
+{
+ // Set up a master.
+ Try<PID<Master>> master = StartMaster();
+ ASSERT_SOME(master);
+
+ // Schedule two machines.
+ maintenance::Schedule schedule = createSchedule(
+ {createWindow({machine1, machine2}, unavailability)});
+
+ Future<Response> response = process::http::post(
+ master.get(),
+ "maintenance/schedule",
+ headers,
+ stringify(JSON::Protobuf(schedule)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+
+ // Deactivate machine1.
+ MachineIDs machines = createMachineList({machine1});
+ response = process::http::post(
+ master.get(),
+ "machine/down",
+ headers,
+ stringify(JSON::Protobuf(machines)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+
+ // Try to unschedule machine1, by posting a schedule without it.
+ schedule = createSchedule({createWindow({machine2}, unavailability)});
+
+ response = process::http::post(
+ master.get(),
+ "maintenance/schedule",
+ headers,
+ stringify(JSON::Protobuf(schedule)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(BadRequest().status, response);
+
+ // Reactivate machine1.
+ machines = createMachineList({machine1});
+ response = process::http::post(
+ master.get(),
+ "machine/up",
+ headers,
+ stringify(JSON::Protobuf(machines)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+}
+
+
// Posts valid and invalid machines to the maintenance start endpoint.
TEST_F(MasterMaintenanceTest, BringDownMachines)
{
@@ -448,6 +500,93 @@ TEST_F(MasterMaintenanceTest, BringUpMachines)
ASSERT_EQ(0, masterSchedule.get().windows().size());
}
+
+// Queries for machine statuses in between maintenance mode transitions.
+TEST_F(MasterMaintenanceTest, MachineStatus)
+{
+ // Set up a master.
+ Try<PID<Master>> master = StartMaster();
+ ASSERT_SOME(master);
+
+ // Try to stop maintenance on an unscheduled machine.
+ maintenance::Schedule schedule = createSchedule(
+ {createWindow({machine1, machine2}, unavailability)});
+
+ Future<Response> response = process::http::post(
+ master.get(),
+ "maintenance/schedule",
+ headers,
+ stringify(JSON::Protobuf(schedule)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+
+ // Get the maintenance statuses.
+ response = process::http::get(master.get(), "maintenance/status");
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+
+ // Check that both machines are draining.
+ Try<JSON::Object> statuses_ =
+ JSON::parse<JSON::Object>(response.get().body);
+
+ ASSERT_SOME(statuses_);
+ Try<maintenance::ClusterStatus> statuses =
+ ::protobuf::parse<maintenance::ClusterStatus>(statuses_.get());
+
+ ASSERT_SOME(statuses);
+ ASSERT_EQ(2, statuses.get().draining_machines().size());
+ ASSERT_EQ(0, statuses.get().down_machines().size());
+
+ // Deactivate machine1.
+ MachineIDs machines = createMachineList({machine1});
+ response = process::http::post(
+ master.get(),
+ "machine/down",
+ headers,
+ stringify(JSON::Protobuf(machines)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+
+ // Get the maintenance statuses.
+ response = process::http::get(master.get(), "maintenance/status");
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+
+ // Check one machine is deactivated.
+ statuses_ = JSON::parse<JSON::Object>(response.get().body);
+
+ ASSERT_SOME(statuses_);
+ statuses = ::protobuf::parse<maintenance::ClusterStatus>(statuses_.get());
+
+ ASSERT_SOME(statuses);
+ ASSERT_EQ(1, statuses.get().draining_machines().size());
+ ASSERT_EQ(1, statuses.get().down_machines().size());
+ ASSERT_EQ("Machine1", statuses.get().down_machines(0).hostname());
+
+ // Reactivate machine1.
+ machines = createMachineList({machine1});
+ response = process::http::post(
+ master.get(),
+ "machine/up",
+ headers,
+ stringify(JSON::Protobuf(machines)));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+
+ // Get the maintenance statuses.
+ response = process::http::get(master.get(), "maintenance/status");
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+
+ // Check that only one machine remains.
+ statuses_ = JSON::parse<JSON::Object>(response.get().body);
+
+ ASSERT_SOME(statuses_);
+ statuses = ::protobuf::parse<maintenance::ClusterStatus>(statuses_.get());
+
+ ASSERT_SOME(statuses);
+ ASSERT_EQ(1, statuses.get().draining_machines().size());
+ ASSERT_EQ(0, statuses.get().down_machines().size());
+ ASSERT_EQ("0.0.0.2", statuses.get().draining_machines(0).ip());
+}
+
} // namespace tests {
} // namespace internal {
} // namespace mesos {
[2/8] mesos git commit: Maintenance Primitives: Added
maintenance-related, registry protobufs.
Posted by jo...@apache.org.
Maintenance Primitives: Added maintenance-related, registry protobufs.
New protobufs:
* MachineID - Describes a single box that holds one or more agents.
* MachineIDs - A list of boxes.
* MachineInfo - Holds a box and some extra information about its status.
* MachineInfo::Mode - An enum for the three states of a machine: UP,
DRAINING, DOWN.
* maintenance::Window - A set of machines and a planned downtime period.
* maintenance::Schedule - A set of maintenance windows.
Review: https://reviews.apache.org/r/36571
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/50344bca
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/50344bca
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/50344bca
Branch: refs/heads/master
Commit: 50344bcaea8b462c1bb096ba46b1d1401b26ba3a
Parents: 7d7f42f
Author: Joseph Wu <jo...@mesosphere.io>
Authored: Sun Aug 30 13:55:10 2015 -0400
Committer: Joris Van Remoortere <jo...@gmail.com>
Committed: Mon Aug 31 13:09:26 2015 -0400
----------------------------------------------------------------------
include/mesos/maintenance/maintenance.hpp | 25 +++++++++
include/mesos/maintenance/maintenance.proto | 67 ++++++++++++++++++++++++
include/mesos/mesos.proto | 59 +++++++++++++++++++++
include/mesos/v1/mesos.proto | 59 +++++++++++++++++++++
src/Makefile.am | 20 +++++++
src/master/registry.proto | 27 ++++++++++
6 files changed, 257 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/50344bca/include/mesos/maintenance/maintenance.hpp
----------------------------------------------------------------------
diff --git a/include/mesos/maintenance/maintenance.hpp b/include/mesos/maintenance/maintenance.hpp
new file mode 100644
index 0000000..7fec3ff
--- /dev/null
+++ b/include/mesos/maintenance/maintenance.hpp
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __MAINTENANCE_PROTO_HPP__
+#define __MAINTENANCE_PROTO_HPP__
+
+// ONLY USEFUL AFTER RUNNING PROTOC.
+#include <mesos/maintenance/maintenance.pb.h>
+
+#endif // __MAINTENANCE_PROTO_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/50344bca/include/mesos/maintenance/maintenance.proto
----------------------------------------------------------------------
diff --git a/include/mesos/maintenance/maintenance.proto b/include/mesos/maintenance/maintenance.proto
new file mode 100644
index 0000000..ba28d0f
--- /dev/null
+++ b/include/mesos/maintenance/maintenance.proto
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import "mesos/mesos.proto";
+
+package mesos.maintenance;
+
+option java_package = "org.apache.mesos.maintenance";
+option java_outer_classname = "Protos";
+
+// This is an illustration of a maintenance `Schedule`:
+//
+// This is a `Window`.
+// Machine ^ |
+// ... | v
+// 12 | +----------------+
+// 11 | | |
+// 10 | +----------------+
+// 9 | +----------------+
+// 8 | | |
+// 7 | +----------------+
+// 6 | +----------------+
+// 5 | | |
+// 4 | +----------------+
+// 3 | +-----------+
+// 2 | | |
+// 1 | +-----------+
+// |
+// +-----------------------------------------------------~~~~->
+// Downtime for maintenance
+
+
+/**
+ * A set of machines scheduled to go into maintenance
+ * in the same `unavailability`.
+ */
+message Window {
+ // Machines affected by this maintenance window.
+ repeated MachineID machine_ids = 1;
+
+ // Interval during which this set of machines is expected to be down.
+ required Unavailability unavailability = 2;
+}
+
+
+/**
+ * A list of maintenance windows.
+ * For example, this may represent a rolling restart of agents.
+ */
+message Schedule {
+ repeated Window windows = 1;
+}
http://git-wip-us.apache.org/repos/asf/mesos/blob/50344bca/include/mesos/mesos.proto
----------------------------------------------------------------------
diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto
index 1e07159..b1deed4 100644
--- a/include/mesos/mesos.proto
+++ b/include/mesos/mesos.proto
@@ -155,6 +155,65 @@ message Unavailability {
/**
+ * Represents a single machine, which may hold one or more slaves.
+ *
+ * NOTE: In order to match a slave to a machine, both the `hostname` and
+ * `ip` must match the values advertised by the slave to the master.
+ * Hostname is not case-sensitive.
+ */
+message MachineID {
+ optional string hostname = 1;
+ optional string ip = 2;
+}
+
+
+/**
+ * A list of machines.
+ *
+ * TODO(josephw): Remove this when https://reviews.apache.org/r/37826/
+ * is submitted.
+ */
+message MachineIDs {
+ repeated MachineID values = 1;
+}
+
+
+/**
+ * Holds information about a single machine, its `mode`, and any other
+ * relevant information which may affect the behavior of the machine.
+ */
+message MachineInfo {
+ // Describes the several states that a machine can be in. A `Mode`
+ // applies to a machine and to all associated slaves on the machine.
+ enum Mode {
+ // In this mode, a machine is behaving normally;
+ // offering resources, executing tasks, etc.
+ UP = 1;
+
+ // In this mode, all slaves on the machine are expected to cooperate with
+ // frameworks to drain resources. In general, draining is done ahead of
+ // a pending `unavailability`. The resources should be drained so as to
+ // maximize utilization prior to the maintenance but without knowingly
+ // violating the frameworks' requirements.
+ DRAINING = 2;
+
+ // In this mode, a machine is not running any tasks and will not offer
+ // any of its resources. Slaves on the machine will not be allowed to
+ // register with the master.
+ DOWN = 3;
+ }
+
+ required MachineID id = 1;
+ optional Mode mode = 2;
+
+ // Signifies that the machine may be unavailable during the given interval.
+ // See comments in `Unavailability` and for the `unavailability` fields
+ // in `Offer` and `InverseOffer` for more information.
+ optional Unavailability unavailability = 3;
+}
+
+
+/**
* Describes a framework.
*/
message FrameworkInfo {
http://git-wip-us.apache.org/repos/asf/mesos/blob/50344bca/include/mesos/v1/mesos.proto
----------------------------------------------------------------------
diff --git a/include/mesos/v1/mesos.proto b/include/mesos/v1/mesos.proto
index 3ac1582..fdd678b 100644
--- a/include/mesos/v1/mesos.proto
+++ b/include/mesos/v1/mesos.proto
@@ -155,6 +155,65 @@ message Unavailability {
/**
+ * Represents a single machine, which may hold one or more agents.
+ *
+ * NOTE: In order to match an agent to a machine, both the `hostname` and
+ * `ip` must match the values advertised by the agent to the master.
+ * Hostname is not case-sensitive.
+ */
+message MachineID {
+ optional string hostname = 1;
+ optional string ip = 2;
+}
+
+
+/**
+ * A list of machines.
+ *
+ * TODO(josephw): Remove this when https://reviews.apache.org/r/37826/
+ * is submitted.
+ */
+message MachineIDs {
+ repeated MachineID values = 1;
+}
+
+
+/**
+ * Holds information about a single machine, its `mode`, and any other
+ * relevant information which may affect the behavior of the machine.
+ */
+message MachineInfo {
+ // Describes the several states that a machine can be in. A `Mode`
+ // applies to a machine and to all associated agents on the machine.
+ enum Mode {
+ // In this mode, a machine is behaving normally;
+ // offering resources, executing tasks, etc.
+ UP = 1;
+
+ // In this mode, all agents on the machine are expected to cooperate with
+ // frameworks to drain resources. In general, draining is done ahead of
+ // a pending `unavailability`. The resources should be drained so as to
+ // maximize utilization prior to the maintenance but without knowingly
+ // violating the frameworks' requirements.
+ DRAINING = 2;
+
+ // In this mode, a machine is not running any tasks and will not offer
+ // any of its resources. agents on the machine will not be allowed to
+ // register with the master.
+ DOWN = 3;
+ }
+
+ required MachineID id = 1;
+ optional Mode mode = 2;
+
+ // Signifies that the machine may be unavailable during the given interval.
+ // See comments in `Unavailability` and for the `unavailability` fields
+ // in `Offer` and `InverseOffer` for more information.
+ optional Unavailability unavailability = 3;
+}
+
+
+/**
* Describes a framework.
*/
message FrameworkInfo {
http://git-wip-us.apache.org/repos/asf/mesos/blob/50344bca/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 7b620ff..3ea3c8d 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -153,6 +153,9 @@ EXECUTOR_PROTO = \
FETCHER_PROTO = \
$(top_srcdir)/include/mesos/fetcher/fetcher.proto
+MAINTENANCE_PROTO = \
+ $(top_srcdir)/include/mesos/maintenance/maintenance.proto
+
ALLOCATOR_PROTO = \
$(top_srcdir)/include/mesos/master/allocator.proto
@@ -184,6 +187,8 @@ CXX_PROTOS = \
../include/mesos/executor/executor.pb.h \
fetcher/fetcher.pb.cc \
../include/mesos/fetcher/fetcher.pb.h \
+ maintenance/maintenance.pb.cc \
+ ../include/mesos/maintenance/maintenance.pb.h \
master/allocator.pb.cc \
../include/mesos/master/allocator.pb.h \
module/module.pb.cc \
@@ -278,6 +283,12 @@ fetcher/%.pb.cc ../include/mesos/fetcher/%.pb.h: $(FETCHER_PROTO)
$(PROTOC) $(PROTOCFLAGS) --cpp_out=../include $^
mv ../include/mesos/fetcher/*.pb.cc $(@D)
+maintenance/%.pb.cc ../include/mesos/maintenance/%.pb.h: $(MAINTENANCE_PROTO)
+ $(MKDIR_P) $(@D)
+ $(MKDIR_P) ../include/mesos/maintenance
+ $(PROTOC) $(PROTOCFLAGS) --cpp_out=../include $^
+ mv ../include/mesos/maintenance/*.pb.cc $(@D)
+
master/%.pb.cc ../include/mesos/master/%.pb.h: $(ALLOCATOR_PROTO)
$(MKDIR_P) $(@D)
$(MKDIR_P) ../include/mesos/master
@@ -538,6 +549,14 @@ fetcher_HEADERS = \
nodist_fetcher_HEADERS = ../include/mesos/fetcher/fetcher.pb.h
+maintenancedir = $(pkgincludedir)/maintenance
+
+maintenance_HEADERS = \
+ $(top_srcdir)/include/mesos/maintenance/maintenance.hpp \
+ $(top_srcdir)/include/mesos/maintenance/maintenance.proto
+
+nodist_maintenance_HEADERS = ../include/mesos/maintenance/maintenance.pb.h
+
masterdir = $(pkgincludedir)/master
master_HEADERS = \
@@ -896,6 +915,7 @@ libmesos_la_SOURCES = \
$(ALLOCATOR_PROTO) \
$(CONTAINERIZER_PROTO) \
$(FETCHER_PROTO) \
+ $(MAINTENANCE_PROTO) \
$(MESOS_PROTO) \
$(MODULE_PROTO) \
$(SCHEDULER_PROTO) \
http://git-wip-us.apache.org/repos/asf/mesos/blob/50344bca/src/master/registry.proto
----------------------------------------------------------------------
diff --git a/src/master/registry.proto b/src/master/registry.proto
index a1995e5..f5de77f 100644
--- a/src/master/registry.proto
+++ b/src/master/registry.proto
@@ -17,10 +17,19 @@
*/
import "mesos/mesos.proto";
+import "mesos/maintenance/maintenance.proto";
package mesos.internal;
+/**
+ * A top level object that is managed by the Registrar and persisted in a
+ * replicated log. This object is recovered upon master startup and failover.
+ */
message Registry {
+ // NOTE: This object defines wrappers around existing objects in case
+ // the Registry wishes to store more information about the wrapped objects
+ // in the future.
+
message Master {
required MasterInfo info = 1;
}
@@ -33,9 +42,27 @@ message Registry {
repeated Slave slaves = 1;
}
+ message Machine {
+ required MachineInfo info = 1;
+ }
+
+ message Machines {
+ repeated Machine machines = 1;
+ }
+
// Most recent leading master.
optional Master master = 1;
// All admitted slaves.
optional Slaves slaves = 2;
+
+ // Holds a list of machines and some status information about each.
+ // See comments in `MachineInfo` for more information.
+ optional Machines machines = 3;
+
+ // Describes a schedule for taking down specific machines for maintenance.
+ // The schedule is meant to give hints to frameworks about potential
+ // unavailability of resources. The `schedules` are related to the status
+ // information found in `machines`.
+ repeated maintenance.Schedule schedules = 4;
}
[3/8] mesos git commit: Maintenance Primitives: Updated master local
state upon recovery.
Posted by jo...@apache.org.
Maintenance Primitives: Updated master local state upon recovery.
Note: Tests will be added with the related HTTP endpoints and registrar
operations, later in this review chain.
Review: https://reviews.apache.org/r/37314
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/691a40e7
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/691a40e7
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/691a40e7
Branch: refs/heads/master
Commit: 691a40e7e1c9d5db8d1869d0f61b7bac7bfdc915
Parents: 50344bc
Author: Joseph Wu <jo...@mesosphere.io>
Authored: Sun Aug 30 13:55:21 2015 -0400
Committer: Joris Van Remoortere <jo...@gmail.com>
Committed: Mon Aug 31 13:09:33 2015 -0400
----------------------------------------------------------------------
include/mesos/type_utils.hpp | 33 +++++++++++++++++++++++++++++++++
src/master/master.cpp | 10 ++++++++++
src/master/master.hpp | 11 +++++++++++
3 files changed, 54 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/691a40e7/include/mesos/type_utils.hpp
----------------------------------------------------------------------
diff --git a/include/mesos/type_utils.hpp b/include/mesos/type_utils.hpp
index 1c8f95b..4fb0037 100644
--- a/include/mesos/type_utils.hpp
+++ b/include/mesos/type_utils.hpp
@@ -31,6 +31,7 @@
#include <stout/hashmap.hpp>
#include <stout/stringify.hpp>
+#include <stout/strings.hpp>
#include <stout/uuid.hpp>
// This file includes definitions for operators on public protobuf
@@ -143,6 +144,21 @@ inline bool operator==(const TaskID& left, const std::string& right)
}
+/**
+ * For machines to match, both the `hostname` and `ip` must be equivalent.
+ * Hostname is not case sensitive, so it is lowercased before comparison.
+ */
+inline bool operator==(const MachineID& left, const MachineID& right)
+{
+ // NOTE: Both fields default to the empty string if they are not specified,
+ // so the string comparisons are safe.
+ return left.has_hostname() == right.has_hostname() &&
+ strings::lower(left.hostname()) == strings::lower(right.hostname()) &&
+ left.has_ip() == right.has_ip() &&
+ left.ip() == right.ip();
+}
+
+
inline bool operator!=(const ContainerID& left, const ContainerID& right)
{
return left.value() != right.value();
@@ -570,6 +586,23 @@ struct hash<std::pair<mesos::FrameworkID, mesos::ExecutorID>>
}
};
+
+template <>
+struct hash<mesos::MachineID>
+{
+ typedef size_t result_type;
+
+ typedef mesos::MachineID argument_type;
+
+ result_type operator()(const argument_type& machineId) const
+ {
+ size_t seed = 0;
+ boost::hash_combine(seed, strings::lower(machineId.hostname()));
+ boost::hash_combine(seed, machineId.ip());
+ return seed;
+ }
+};
+
} // namespace std {
#endif // __MESOS_TYPE_UTILS_H__
http://git-wip-us.apache.org/repos/asf/mesos/blob/691a40e7/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index b824f96..564fbcb 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -1331,6 +1331,16 @@ Future<Nothing> Master::_recover(const Registry& registry)
&Self::recoveredSlavesTimeout,
registry);
+ // Save the maintenance schedule.
+ foreach (const mesos::maintenance::Schedule& schedule, registry.schedules()) {
+ maintenance.schedules.push_back(schedule);
+ }
+
+ // Save the machine info for each machine.
+ foreach (const Registry::Machine& machine, registry.machines().machines()) {
+ machineInfos[machine.info().id()] = machine.info();
+ }
+
// Recovery is now complete!
LOG(INFO) << "Recovered " << registry.slaves().slaves().size() << " slaves"
<< " from the Registry (" << Bytes(registry.ByteSize()) << ")"
http://git-wip-us.apache.org/repos/asf/mesos/blob/691a40e7/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index 36c6759..edcbeed 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -28,6 +28,7 @@
#include <boost/circular_buffer.hpp>
+#include <mesos/maintenance/maintenance.hpp>
#include <mesos/mesos.hpp>
#include <mesos/resources.hpp>
#include <mesos/scheduler/scheduler.hpp>
@@ -888,6 +889,16 @@ private:
MasterInfo info_;
+ // Holds some info which affects how a machine behaves.
+ // See the `MachineInfo` protobuf for more information.
+ hashmap<MachineID, MachineInfo> machineInfos;
+
+ struct Maintenance
+ {
+ // Holds the maintenance schedule, as given by the operator.
+ std::list<mesos::maintenance::Schedule> schedules;
+ } maintenance;
+
// Indicates when recovery is complete. Recovery begins once the
// master is elected as a leader.
Option<process::Future<Nothing>> recovered;
[4/8] mesos git commit: Maintenance Primitives: Added maintenance
proto helpers.
Posted by jo...@apache.org.
Maintenance Primitives: Added maintenance proto helpers.
This includes helpers for constructing (used in tests) and for
validation (used in HTTP endpoints).
Review: https://reviews.apache.org/r/37900
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/8e43aba0
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/8e43aba0
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/8e43aba0
Branch: refs/heads/master
Commit: 8e43aba00165be0718200755c043fcce1b5ee93e
Parents: 691a40e
Author: Joseph Wu <jo...@mesosphere.io>
Authored: Sun Aug 30 13:55:42 2015 -0400
Committer: Joris Van Remoortere <jo...@gmail.com>
Committed: Mon Aug 31 13:09:40 2015 -0400
----------------------------------------------------------------------
src/Makefile.am | 2 +
src/common/protobuf_utils.cpp | 58 ++++++++++++++
src/common/protobuf_utils.hpp | 44 +++++++++++
src/master/maintenance.cpp | 157 +++++++++++++++++++++++++++++++++++++
src/master/maintenance.hpp | 79 +++++++++++++++++++
src/master/registrar.hpp | 10 ++-
6 files changed, 348 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/8e43aba0/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 3ea3c8d..1b07af4 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -439,6 +439,7 @@ libmesos_no_3rdparty_la_SOURCES = \
master/detector.cpp \
master/flags.cpp \
master/http.cpp \
+ master/maintenance.cpp \
master/master.cpp \
master/metrics.cpp \
master/registry.hpp \
@@ -727,6 +728,7 @@ libmesos_no_3rdparty_la_SOURCES += \
master/constants.hpp \
master/detector.hpp \
master/flags.hpp \
+ master/maintenance.hpp \
master/master.hpp \
master/metrics.hpp \
master/repairer.hpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/8e43aba0/src/common/protobuf_utils.cpp
----------------------------------------------------------------------
diff --git a/src/common/protobuf_utils.cpp b/src/common/protobuf_utils.cpp
index 6b28355..0861270 100644
--- a/src/common/protobuf_utils.cpp
+++ b/src/common/protobuf_utils.cpp
@@ -248,6 +248,64 @@ ContainerState createContainerState(
} // namespace slave {
+namespace maintenance {
+
+Unavailability createUnavailability(
+ const process::Time& start,
+ const Option<Duration>& duration)
+{
+ Unavailability unavailability;
+ unavailability.mutable_start()->set_nanoseconds(start.duration().ns());
+
+ if (duration.isSome()) {
+ unavailability.mutable_duration()->set_nanoseconds(duration.get().ns());
+ }
+
+ return unavailability;
+}
+
+
+MachineIDs createMachineList(std::initializer_list<MachineID> ids)
+{
+ MachineIDs array;
+
+ foreach (const MachineID& id, ids) {
+ array.add_values()->CopyFrom(id);
+ }
+
+ return array;
+}
+
+
+mesos::maintenance::Window createWindow(
+ std::initializer_list<MachineID> ids,
+ const Unavailability& unavailability)
+{
+ mesos::maintenance::Window window;
+ window.mutable_unavailability()->CopyFrom(unavailability);
+
+ foreach (const MachineID& id, ids) {
+ window.add_machine_ids()->CopyFrom(id);
+ }
+
+ return window;
+}
+
+
+mesos::maintenance::Schedule createSchedule(
+ std::initializer_list<mesos::maintenance::Window> windows)
+{
+ mesos::maintenance::Schedule schedule;
+
+ foreach (const mesos::maintenance::Window& window, windows) {
+ schedule.add_windows()->CopyFrom(window);
+ }
+
+ return schedule;
+}
+
+} // namespace maintenance {
+
} // namespace protobuf {
} // namespace internal {
} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/8e43aba0/src/common/protobuf_utils.hpp
----------------------------------------------------------------------
diff --git a/src/common/protobuf_utils.hpp b/src/common/protobuf_utils.hpp
index 63eeb77..86474ea 100644
--- a/src/common/protobuf_utils.hpp
+++ b/src/common/protobuf_utils.hpp
@@ -19,13 +19,20 @@
#ifndef __PROTOBUF_UTILS_HPP__
#define __PROTOBUF_UTILS_HPP__
+#include <initializer_list>
#include <string>
#include <mesos/mesos.hpp>
+#include <mesos/maintenance/maintenance.hpp>
+
#include <mesos/slave/isolator.hpp>
+#include <process/time.hpp>
+
+#include <stout/duration.hpp>
#include <stout/ip.hpp>
+#include <stout/none.hpp>
#include <stout/option.hpp>
#include <stout/uuid.hpp>
@@ -92,6 +99,43 @@ mesos::slave::ContainerState createContainerState(
} // namespace slave {
+namespace maintenance {
+
+/**
+ * Helper for constructing an unavailability from a `Time` and `Duration`.
+ */
+Unavailability createUnavailability(
+ const process::Time& start,
+ const Option<Duration>& duration = None());
+
+
+/**
+ * Helper for constructing a list of `MachineID`.
+ *
+ * TODO(josephw): Remove this when https://reviews.apache.org/r/37826/
+ * is submitted.
+ */
+MachineIDs createMachineList(std::initializer_list<MachineID> ids);
+
+
+/**
+ * Helper for constructing a maintenance `Window`.
+ * See `createUnavailability` above.
+ */
+mesos::maintenance::Window createWindow(
+ std::initializer_list<MachineID> ids,
+ const Unavailability& unavailability);
+
+
+/**
+ * Helper for constructing a maintenance `Schedule`.
+ * See `createWindow` above.
+ */
+mesos::maintenance::Schedule createSchedule(
+ std::initializer_list<mesos::maintenance::Window> windows);
+
+} // namespace maintenance {
+
} // namespace protobuf {
} // namespace internal {
} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/8e43aba0/src/master/maintenance.cpp
----------------------------------------------------------------------
diff --git a/src/master/maintenance.cpp b/src/master/maintenance.cpp
new file mode 100644
index 0000000..221dd02
--- /dev/null
+++ b/src/master/maintenance.cpp
@@ -0,0 +1,157 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <mesos/type_utils.hpp>
+
+#include <mesos/maintenance/maintenance.hpp>
+
+#include <stout/duration.hpp>
+#include <stout/error.hpp>
+#include <stout/hashset.hpp>
+#include <stout/ip.hpp>
+#include <stout/nothing.hpp>
+#include <stout/strings.hpp>
+
+#include "master/maintenance.hpp"
+
+namespace mesos {
+namespace internal {
+namespace master {
+namespace maintenance {
+
+using namespace mesos::maintenance;
+
+namespace validation {
+
+Try<Nothing> schedule(
+ const maintenance::Schedule& schedule,
+ const hashmap<MachineID, MachineInfo>& infos)
+{
+ hashset<MachineID> updated;
+ foreach (const maintenance::Window& window, schedule.windows()) {
+ // Check that each window has at least one machine.
+ if (window.machine_ids().size() == 0) {
+ return Error("List of machines in the maintenance window is empty");
+ }
+
+ // Check the time specified by the unavailability.
+ Try<Nothing> unavailability =
+ maintenance::validation::unavailability(window.unavailability());
+
+ if (unavailability.isError()) {
+ return Error(unavailability.error());
+ }
+
+ // Collect machines from the updated schedule into a set.
+ foreach (const MachineID& id, window.machine_ids()) {
+ // Validate the single machine.
+ Try<Nothing> validId = validation::machine(id);
+ if (validId.isError()) {
+ return Error(validId.error());
+ }
+
+ // Check that the machine is unique.
+ if (updated.contains(id)) {
+ return Error(
+ "Machine '" + id.DebugString() +
+ "' appears more than once in the schedule");
+ }
+
+ updated.insert(id);
+ }
+ }
+
+ // Ensure that no `DOWN` machine is removed from the schedule.
+ foreachpair (const MachineID& id, const MachineInfo& info, infos) {
+ if (info.mode() == MachineInfo::DOWN && !updated.contains(id)) {
+ return Error(
+ "Machine '" + id.DebugString() +
+ "' is deactivated and cannot be removed from the schedule");
+ }
+ }
+
+ return Nothing();
+}
+
+
+Try<Nothing> unavailability(const Unavailability& unavailability)
+{
+ const Duration duration =
+ Nanoseconds(unavailability.duration().nanoseconds());
+
+ // Check the bounds of the unavailability.
+ if (duration < Duration::zero()) {
+ return Error("Unavailability 'duration' is negative");
+ }
+
+ return Nothing();
+}
+
+
+Try<Nothing> machines(const MachineIDs& ids)
+{
+ if (ids.values().size() <= 0) {
+ return Error("List of machines is empty");
+ }
+
+ // Verify that the machine has at least one non-empty field value.
+ hashset<MachineID> uniques;
+ foreach (const MachineID& id, ids.values()) {
+ // Validate the single machine.
+ Try<Nothing> validId = validation::machine(id);
+ if (validId.isError()) {
+ return Error(validId.error());
+ }
+
+ // Check machine uniqueness.
+ if (uniques.contains(id)) {
+ return Error(
+ "Machine '" + id.DebugString() +
+ "' appears more than once in the schedule");
+ }
+
+ uniques.insert(id);
+ }
+
+ return Nothing();
+}
+
+
+Try<Nothing> machine(const MachineID& id)
+{
+ // Verify that the machine has at least one non-empty field value.
+ if (id.hostname().empty() && id.ip().empty()) {
+ return Error("Both 'hostname' and 'ip' for a machine are empty");
+ }
+
+ // Validate the IP field.
+ if (!id.ip().empty()) {
+ Try<net::IP> ip = net::IP::parse(id.ip(), AF_INET);
+ if (ip.isError()) {
+ return Error(ip.error());
+ }
+ }
+
+ return Nothing();
+}
+
+} // namespace validation {
+} // namespace maintenance {
+} // namespace master {
+} // namespace internal {
+} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/8e43aba0/src/master/maintenance.hpp
----------------------------------------------------------------------
diff --git a/src/master/maintenance.hpp b/src/master/maintenance.hpp
new file mode 100644
index 0000000..833665e
--- /dev/null
+++ b/src/master/maintenance.hpp
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __MESOS_MASTER_MAINTENANCE_HPP__
+#define __MESOS_MASTER_MAINTENANCE_HPP__
+
+#include <mesos/mesos.hpp>
+
+#include <mesos/maintenance/maintenance.hpp>
+
+#include <stout/hashset.hpp>
+#include <stout/nothing.hpp>
+#include <stout/try.hpp>
+
+#include "master/registrar.hpp"
+#include "master/registry.hpp"
+
+namespace mesos {
+namespace internal {
+namespace master {
+namespace maintenance {
+namespace validation {
+
+/**
+ * Performs the following checks on the new maintenance schedule:
+ * - Each window in the new schedule has at least one machine.
+ * - All unavailabilities adhere to the `unavailability` method below.
+ * - Each machine appears in the schedule once and only once.
+ * - All currently `DOWN` machines are present in the schedule.
+ * - All checks in the `machine` method below.
+ */
+Try<Nothing> schedule(
+ const mesos::maintenance::Schedule& schedule,
+ const hashmap<MachineID, MachineInfo>& infos);
+
+
+// Checks that the `duration` of the unavailability is non-negative.
+Try<Nothing> unavailability(
+ const Unavailability& unavailability);
+
+
+/**
+ * Performs the following checks on a list of machines:
+ * - Each machine appears in the list once and only once.
+ * - The list is non-empty.
+ * - All checks in the `machine` method below.
+ */
+Try<Nothing> machines(const MachineIDs& ids);
+
+
+/**
+ * Performs the following checks on a single machine:
+ * - The machine has at least a hostname or IP.
+ * - IP is correctly formed.
+ */
+Try<Nothing> machine(const MachineID& id);
+
+} // namespace validation {
+} // namespace maintenance {
+} // namespace master {
+} // namespace internal {
+} // namespace mesos {
+
+#endif // __MESOS_MASTER_MAINTENANCE_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/8e43aba0/src/master/registrar.hpp
----------------------------------------------------------------------
diff --git a/src/master/registrar.hpp b/src/master/registrar.hpp
index c6a0655..d464f61 100644
--- a/src/master/registrar.hpp
+++ b/src/master/registrar.hpp
@@ -49,8 +49,14 @@ public:
Operation() : success(false) {}
virtual ~Operation() {}
- // Attempts to invoke the operation on 'registry' (and the
- // accumulators, in this case 'slaveIDs').
+ // Attempts to invoke the operation on the registry object.
+ // Aided by accumulator(s):
+ // slaveIDs - is the set of registered slaves.
+ //
+ // NOTE: the "strict" parameter only applies to operations that
+ // affect slaves (i.e. registration). See Flags::registry_strict
+ // in master/flags.cpp for more information.
+ //
// Returns whether the operation mutates 'registry', or an error if
// the operation cannot be applied successfully.
Try<bool> operator()(