You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by bm...@apache.org on 2014/05/02 19:56:41 UTC
git commit: Updated the Registrar to abort permanently upon
encountering a Failure.
Repository: mesos
Updated Branches:
refs/heads/master 3657550ec -> 5eaf1eb34
Updated the Registrar to abort permanently upon encountering a Failure.
It's possible for a backed-up master (many items in its queue) to have
many operations enqueued in the Registrar.
In this event, the Master won't commit suicide until the initial
failure is processed. However, in the interim, subsequent operations
are potentially being performed against the Registrar. This could lead
to fighting between masters if a "demoted" master re-attempts to
acquirelog-leadership! This scenario can occur if the "demoted" master
has a large queue and the demotion event is towards the back of the
master's queue.
It would be preferable to ensure that after losing log leadership,
the "demoted" master does not try to re-acquire log leadership and
write to the log.
Review: https://reviews.apache.org/r/20981
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/5eaf1eb3
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/5eaf1eb3
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/5eaf1eb3
Branch: refs/heads/master
Commit: 5eaf1eb346fc2f46c852c1246bdff12a89216b60
Parents: 3657550
Author: Ben Mahler <be...@gmail.com>
Authored: Fri May 2 10:49:11 2014 -0700
Committer: Benjamin Mahler <bm...@twitter.com>
Committed: Fri May 2 10:56:37 2014 -0700
----------------------------------------------------------------------
src/master/registrar.cpp | 89 +++++++++++++++++++++++++++-----------
src/tests/registrar_tests.cpp | 73 ++++++++++++++++++++-----------
2 files changed, 111 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/5eaf1eb3/src/master/registrar.cpp
----------------------------------------------------------------------
diff --git a/src/master/registrar.cpp b/src/master/registrar.cpp
index fecc314..5d9dfed 100644
--- a/src/master/registrar.cpp
+++ b/src/master/registrar.cpp
@@ -175,6 +175,12 @@ private:
const Future<Option<Variable<Registry> > >& store,
deque<Owned<Operation> > operations);
+ // Fails all pending operations and transitions the Registrar
+ // into an error state in which all subsequent operations will fail.
+ // This ensures we don't attempt to re-acquire log leadership by
+ // performing more State storage operations.
+ void abort(const string& message);
+
Option<Variable<Registry> > variable;
deque<Owned<Operation> > operations;
bool updating; // Used to signify fetching (recovering) or storing.
@@ -184,6 +190,10 @@ private:
// Used to compose our operations with recovery.
Option<Owned<Promise<Registry> > > recovered;
+
+ // When an error is encountered from abort(), we'll fail all
+ // subsequent operations.
+ Option<Error> error;
};
@@ -201,6 +211,18 @@ Future<T> timeout(
}
+// Helper for failing a deque of operations.
+void fail(deque<Owned<Operation> >* operations, const string& message)
+{
+ while (!operations->empty()) {
+ const Owned<Operation>& operation = operations->front();
+ operations->pop_front();
+
+ operation->fail(message);
+ }
+}
+
+
Future<Response> RegistrarProcess::registry(const Request& request)
{
JSON::Object result;
@@ -272,9 +294,9 @@ string RegistrarProcess::registryHelp()
Future<Registry> RegistrarProcess::recover(const MasterInfo& info)
{
- LOG(INFO) << "Recovering registrar";
-
if (recovered.isNone()) {
+ LOG(INFO) << "Recovering registrar";
+
metrics.state_fetch.time(state->fetch<Registry>("registry"))
.after(flags.registry_fetch_timeout,
lambda::bind(
@@ -303,11 +325,12 @@ void RegistrarProcess::_recover(
recovered.get()->fail("Failed to recover registrar: " +
(recovery.isFailed() ? recovery.failure() : "discarded"));
} else {
- LOG(INFO) << "Successfully recovered registrar";
-
// Save the registry.
variable = recovery.get();
+ LOG(INFO) << "Successfully fetched the registry "
+ << "(" << Bytes(variable.get().get().ByteSize()) << ")";
+
// Perform the Recover operation to add the new MasterInfo.
Owned<Operation> operation(new Recover(info));
operations.push_back(operation);
@@ -331,6 +354,8 @@ void RegistrarProcess::__recover(const Future<bool>& recover)
recovered.get()->fail("Failed to recover registrar: "
"Failed to persist MasterInfo: version mismatch");
} else {
+ LOG(INFO) << "Successfully recovered registrar";
+
// At this point _update() has updated 'variable' to contain
// the Registry with the latest MasterInfo.
// Set the promise and un-gate any pending operations.
@@ -353,6 +378,10 @@ Future<bool> RegistrarProcess::apply(Owned<Operation> operation)
Future<bool> RegistrarProcess::_apply(Owned<Operation> operation)
{
+ if (error.isSome()) {
+ return Failure(error.get());
+ }
+
CHECK_SOME(variable);
operations.push_back(operation);
@@ -371,6 +400,7 @@ void RegistrarProcess::update()
}
CHECK(!updating);
+ CHECK(error.isNone());
updating = true;
@@ -413,42 +443,51 @@ void RegistrarProcess::_update(
{
updating = false;
- // Set the variable if the storage operation succeeded.
- if (!store.isReady()) {
- LOG(ERROR) << "Failed to update 'registry': "
- << (store.isFailed() ? store.failure() : "discarded");
- } else if (store.get().isNone()) {
- LOG(WARNING) << "Failed to update 'registry': version mismatch";
- } else {
- LOG(INFO) << "Successfully updated 'registry'";
- variable = store.get().get();
+ // Abort if the storage operation did not succeed.
+ if (!store.isReady() || store.get().isNone()) {
+ string message = "Failed to update 'registry': ";
+
+ if (store.isFailed()) {
+ message += store.failure();
+ } else if (store.isDiscarded()) {
+ message += "discarded";
+ } else {
+ message += "version mismatch";
+ }
+
+ fail(&applied, message);
+ abort(message);
+
+ return;
}
+ LOG(INFO) << "Successfully updated 'registry'";
+ variable = store.get().get();
+
// Remove the operations.
while (!applied.empty()) {
Owned<Operation> operation = applied.front();
applied.pop_front();
- if (!store.isReady()) {
- operation->fail("Failed to update 'registry': " +
- (store.isFailed() ? store.failure() : "discarded"));
- } else {
- if (store.get().isNone()) {
- operation->fail("Failed to update 'registry': version mismatch");
- } else {
- operation->set();
- }
- }
+ operation->set();
}
- applied.clear();
-
if (!operations.empty()) {
update();
}
}
+void RegistrarProcess::abort(const string& message)
+{
+ error = Error(message);
+
+ LOG(ERROR) << "Registrar aborting: " << message;
+
+ fail(&operations, message);
+}
+
+
Registrar::Registrar(const Flags& flags, State* state)
{
process = new RegistrarProcess(flags, state);
http://git-wip-us.apache.org/repos/asf/mesos/blob/5eaf1eb3/src/tests/registrar_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/registrar_tests.cpp b/src/tests/registrar_tests.cpp
index 917a470..cd83bf0 100644
--- a/src/tests/registrar_tests.cpp
+++ b/src/tests/registrar_tests.cpp
@@ -124,6 +124,15 @@ protected:
state = new State(storage);
master.CopyFrom(protobuf::createMasterInfo(UPID("master@127.0.0.1:5050")));
+
+ SlaveID id;
+ id.set_value("1");
+
+ SlaveInfo info;
+ info.set_hostname("localhost");
+ info.mutable_id()->CopyFrom(id);
+
+ slave.CopyFrom(info);
}
virtual void TearDown()
@@ -143,6 +152,7 @@ protected:
Replica* replica2;
MasterInfo master;
+ SlaveInfo slave;
Flags flags;
};
@@ -167,12 +177,6 @@ TEST_P(RegistrarTest, recover)
{
Registrar registrar(flags, state);
- SlaveInfo slave;
- slave.set_hostname("localhost");
- SlaveID id;
- id.set_value("1");
- slave.mutable_id()->CopyFrom(id);
-
// Operations preceding recovery will fail.
AWAIT_EXPECT_FAILED(
registrar.apply(Owned<Operation>(new AdmitSlave(slave))));
@@ -206,19 +210,12 @@ TEST_P(RegistrarTest, admit)
Registrar registrar(flags, state);
AWAIT_READY(registrar.recover(master));
- SlaveInfo info1;
- info1.set_hostname("localhost");
-
- SlaveID id1;
- id1.set_value("1");
- info1.mutable_id()->CopyFrom(id1);
-
- AWAIT_EQ(true, registrar.apply(Owned<Operation>(new AdmitSlave(info1))));
+ AWAIT_EQ(true, registrar.apply(Owned<Operation>(new AdmitSlave(slave))));
if (flags.registry_strict) {
- AWAIT_EQ(false, registrar.apply(Owned<Operation>(new AdmitSlave(info1))));
+ AWAIT_EQ(false, registrar.apply(Owned<Operation>(new AdmitSlave(slave))));
} else {
- AWAIT_EQ(true, registrar.apply(Owned<Operation>(new AdmitSlave(info1))));
+ AWAIT_EQ(true, registrar.apply(Owned<Operation>(new AdmitSlave(slave))));
}
}
@@ -314,13 +311,6 @@ TEST_P(RegistrarTest, remove)
TEST_P(RegistrarTest, bootstrap)
{
- SlaveID id;
- id.set_value("1");
-
- SlaveInfo info;
- info.set_hostname("localhost");
- info.mutable_id()->CopyFrom(id);
-
// Run 1 readmits a slave that is not present.
{
Registrar registrar(flags, state);
@@ -329,10 +319,10 @@ TEST_P(RegistrarTest, bootstrap)
// If not strict, we should be allowed to readmit the slave.
if (flags.registry_strict) {
AWAIT_EQ(false,
- registrar.apply(Owned<Operation>(new ReadmitSlave(info))));
+ registrar.apply(Owned<Operation>(new ReadmitSlave(slave))));
} else {
AWAIT_EQ(true,
- registrar.apply(Owned<Operation>(new ReadmitSlave(info))));
+ registrar.apply(Owned<Operation>(new ReadmitSlave(slave))));
}
}
@@ -348,7 +338,7 @@ TEST_P(RegistrarTest, bootstrap)
EXPECT_EQ(0, registry.get().slaves().slaves().size());
} else {
ASSERT_EQ(1, registry.get().slaves().slaves().size());
- EXPECT_EQ(info, registry.get().slaves().slaves(0).info());
+ EXPECT_EQ(slave, registry.get().slaves().slaves(0).info());
}
}
}
@@ -387,6 +377,9 @@ TEST_P(RegistrarTest, fetchTimeout)
AWAIT_FAILED(recover);
Clock::resume();
+
+ // Ensure the registrar fails subsequent operations.
+ AWAIT_FAILED(registrar.apply(Owned<Operation>(new AdmitSlave(slave))));
}
@@ -416,6 +409,34 @@ TEST_P(RegistrarTest, storeTimeout)
AWAIT_FAILED(recover);
Clock::resume();
+
+ // Ensure the registrar fails subsequent operations.
+ AWAIT_FAILED(registrar.apply(Owned<Operation>(new AdmitSlave(slave))));
+}
+
+
+TEST_P(RegistrarTest, abort)
+{
+ MockStorage storage;
+ State state(&storage);
+
+ Registrar registrar(flags, &state);
+
+ EXPECT_CALL(storage, get(_))
+ .WillOnce(Return(None()));
+
+ EXPECT_CALL(storage, set(_, _))
+ .WillOnce(Return(Future<bool>(true))) // Recovery.
+ .WillOnce(Return(Future<bool>::failed("failure"))) // Failure.
+ .WillRepeatedly(Return(Future<bool>(true))); // Success.
+
+ AWAIT_READY(registrar.recover(master));
+
+ // Storage failure.
+ AWAIT_FAILED(registrar.apply(Owned<Operation>(new AdmitSlave(slave))));
+
+ // The registrar should now be aborted!
+ AWAIT_FAILED(registrar.apply(Owned<Operation>(new AdmitSlave(slave))));
}