You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by vi...@apache.org on 2016/09/19 22:49:54 UTC
[01/27] mesos git commit: Changed master to allow partitioned slaves
to reregister.
Repository: mesos
Updated Branches:
refs/heads/master 34e1240d5 -> 29236068f
Changed master to allow partitioned slaves to reregister.
The previous behavior was to shutdown partitioned agents that attempt to
reregister---unless the master has failed over, in which case the
reregistration is allowed (when running in "non-strict" mode).
The new behavior is always to allow partitioned agents to reregister.
This is part of a longer-term project to allow frameworks to define
their own policies for handling tasks running on partitioned agents.
In particular, if a framework has the PARTITION_AWARE capability, any
tasks running on the partitioned agent will continue to run after
reregistration. If the framework is not PARTITION_AWARE, any tasks that
were running on such an agent will be killed after the agent reregisters
(unless the master has failed over). This is for backward compatibility
with the previous ("non-strict") behavior. Note that regardless of the
PARTITION_AWARE capability, the agent will not be shutdown, which is a
change from the previous Mesos behavior.
This commit also changes the master so that if an agent is removed and
then the master receives a message from that agent, the master will no
longer attempt to shutdown the agent. This is consistent with the goal
of getting the master out of the business of shutting down agents that
we suspect are unhealthy. Such an agent will eventually realize it is
not registered with the master (e.g., because it won't receive any pings
from the master), which will cause it to reregister.
Review: https://reviews.apache.org/r/50705/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/937c85f2
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/937c85f2
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/937c85f2
Branch: refs/heads/master
Commit: 937c85f2f6528d1ac56ea9a7aa174ca0bd371d0c
Parents: 34e1240
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:47:01 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:47:01 2016 -0700
----------------------------------------------------------------------
src/master/master.cpp | 471 ++++++++++++++++++++++++-------------
src/master/master.hpp | 64 ++---
src/tests/master_tests.cpp | 16 +-
src/tests/partition_tests.cpp | 460 ++++++++++++++++++++++++------------
4 files changed, 642 insertions(+), 369 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/937c85f2/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index b88472f..7e501ec 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -190,10 +190,10 @@ protected:
timeouts = 0;
pinged = false;
- // Cancel any pending shutdown.
- if (shuttingDown.isSome()) {
+ // Cancel any pending unreachable transitions.
+ if (markingUnreachable.isSome()) {
// Need a copy for non-const access.
- Future<Nothing> future = shuttingDown.get();
+ Future<Nothing> future = markingUnreachable.get();
future.discard();
}
}
@@ -205,64 +205,64 @@ protected:
if (timeouts >= maxSlavePingTimeouts) {
// No pong has been received for the last
// 'maxSlavePingTimeouts' pings.
- shutdown();
+ markUnreachable();
}
}
- // NOTE: We keep pinging even if we schedule a shutdown. This is
- // because if the slave eventually responds to a ping, we can
- // cancel the shutdown.
+ // NOTE: We keep pinging even if we schedule a transition to
+ // UNREACHABLE. This is because if the slave eventually responds
+ // to a ping, we can cancel the UNREACHABLE transition.
ping();
}
- // NOTE: The shutdown of the slave is rate limited and can be
- // canceled if a pong was received before the actual shutdown is
- // called.
- void shutdown()
+ // Marking slaves unreachable is rate-limited and can be canceled if
+ // a pong is received before `_markUnreachable` is called.
+ //
+ // TODO(neilc): Using a rate-limit when marking slaves unreachable
+ // is only necessary for frameworks that are not PARTITION_AWARE.
+ // For such frameworks, we shutdown their tasks when an unreachable
+ // agent reregisters, so a rate-limit is a useful safety
+ // precaution. Once all frameworks are PARTITION_AWARE, we can
+ // likely remove the rate-limit (MESOS-5948).
+ void markUnreachable()
{
- if (shuttingDown.isSome()) {
- return; // Shutdown is already in progress.
+ if (markingUnreachable.isSome()) {
+ return; // Unreachable transition is already in progress.
}
Future<Nothing> acquire = Nothing();
if (limiter.isSome()) {
- LOG(INFO) << "Scheduling shutdown of agent " << slaveId
- << " due to health check timeout";
+ LOG(INFO) << "Scheduling transition of agent " << slaveId
+ << " to UNREACHABLE because of health check timeout";
acquire = limiter.get()->acquire();
}
- shuttingDown = acquire.onAny(defer(self(), &Self::_shutdown));
+ markingUnreachable = acquire.onAny(defer(self(), &Self::_markUnreachable));
++metrics->slave_unreachable_scheduled;
}
- void _shutdown()
+ void _markUnreachable()
{
- CHECK_SOME(shuttingDown);
+ CHECK_SOME(markingUnreachable);
- const Future<Nothing>& future = shuttingDown.get();
+ const Future<Nothing>& future = markingUnreachable.get();
CHECK(!future.isFailed());
if (future.isReady()) {
- LOG(INFO) << "Shutting down agent " << slaveId
- << " due to health check timeout";
-
++metrics->slave_unreachable_completed;
- dispatch(master,
- &Master::shutdownSlave,
- slaveId,
- "health check timed out");
+ dispatch(master, &Master::markUnreachable, slaveId);
} else if (future.isDiscarded()) {
- LOG(INFO) << "Canceling shutdown of agent " << slaveId
- << " since a pong is received!";
+ LOG(INFO) << "Canceling transition of agent " << slaveId
+ << " to UNREACHABLE because a pong was received!";
++metrics->slave_unreachable_canceled;
}
- shuttingDown = None();
+ markingUnreachable = None();
}
private:
@@ -272,7 +272,7 @@ private:
const PID<Master> master;
const Option<shared_ptr<RateLimiter>> limiter;
shared_ptr<Metrics> metrics;
- Option<Future<Nothing>> shuttingDown;
+ Option<Future<Nothing>> markingUnreachable;
const Duration slavePingTimeout;
const size_t maxSlavePingTimeouts;
uint32_t timeouts;
@@ -1725,7 +1725,7 @@ void Master::recoveredSlavesTimeout(const Registry& registry)
const string failure = "Agent removal rate limit acquisition failed";
acquire
- .then(defer(self(), &Self::removeSlave, slave))
+ .then(defer(self(), &Self::markUnreachableAfterFailover, slave))
.onFailed(lambda::bind(fail, failure, lambda::_1))
.onDiscarded(lambda::bind(fail, failure, "discarded"));
@@ -1734,13 +1734,13 @@ void Master::recoveredSlavesTimeout(const Registry& registry)
}
-Nothing Master::removeSlave(const Registry::Slave& slave)
+Nothing Master::markUnreachableAfterFailover(const Registry::Slave& slave)
{
// The slave is removed from 'recovered' when it re-registers.
if (!slaves.recovered.contains(slave.info().id())) {
- LOG(INFO) << "Canceling removal of agent "
+ LOG(INFO) << "Canceling transition of agent "
<< slave.info().id() << " (" << slave.info().hostname() << ")"
- << " since it re-registered!";
+ << " to unreachable because it re-registered";
++metrics->slave_unreachable_canceled;
return Nothing();
@@ -1749,7 +1749,7 @@ Nothing Master::removeSlave(const Registry::Slave& slave)
LOG(WARNING) << "Agent " << slave.info().id()
<< " (" << slave.info().hostname() << ") did not re-register"
<< " within " << flags.agent_reregister_timeout
- << " after master failover; removing it from the registrar";
+ << " after master failover; marking it unreachable";
++metrics->slave_unreachable_completed;
++metrics->recovery_slave_removals;
@@ -1757,25 +1757,24 @@ Nothing Master::removeSlave(const Registry::Slave& slave)
slaves.recovered.erase(slave.info().id());
if (flags.registry_strict) {
- slaves.removing.insert(slave.info().id());
+ slaves.markingUnreachable.insert(slave.info().id());
- registrar->apply(Owned<Operation>(new RemoveSlave(slave.info())))
+ registrar->apply(Owned<Operation>(new MarkSlaveUnreachable(slave.info())))
.onAny(defer(self(),
- &Self::_removeSlave,
+ &Self::_markUnreachable,
slave.info(),
vector<StatusUpdate>(), // No TASK_LOST updates to send.
lambda::_1,
- "did not re-register after master failover",
- metrics->slave_removals_reason_unhealthy));
+ "did not re-register after master failover"));
} else {
// When a non-strict registry is in use, we want to ensure the
// registry is used in a write-only manner. Therefore we remove
// the slave from the registry but we do not inform the
// framework.
const string& message =
- "Failed to remove agent " + stringify(slave.info().id());
+ "Failed to mark agent " + stringify(slave.info().id()) + " unreachable";
- registrar->apply(Owned<Operation>(new RemoveSlave(slave.info())))
+ registrar->apply(Owned<Operation>(new MarkSlaveUnreachable(slave.info())))
.onFailed(lambda::bind(fail, message, lambda::_1));
}
@@ -4688,18 +4687,14 @@ void Master::executorMessage(
metrics->messages_executor_to_framework++;
if (slaves.removed.get(slaveId).isSome()) {
- // If the slave is removed, we have already informed
- // frameworks that its tasks were LOST, so the slave
- // should shut down.
+ // If the slave has been removed, drop the executor message. The
+ // master is no longer trying to health check this slave; when the
+ // slave realizes it hasn't received any pings from the master, it
+ // will eventually try to reregister.
LOG(WARNING) << "Ignoring executor message"
<< " from executor" << " '" << executorId << "'"
<< " of framework " << frameworkId
- << " on removed agent " << slaveId
- << " ; asking agent to shutdown";
-
- ShutdownMessage message;
- message.set_message("Executor message from unknown agent");
- reply(message);
+ << " on removed agent " << slaveId;
metrics->invalid_executor_to_framework_messages++;
return;
}
@@ -5019,22 +5014,6 @@ void Master::reregisterSlave(
return;
}
- if (slaves.removed.get(slaveInfo.id()).isSome()) {
- // To compensate for the case where a non-strict registrar is
- // being used, we explicitly deny removed slaves from
- // re-registering. This is because a non-strict registrar cannot
- // enforce this. We've already told frameworks the tasks were
- // lost so it's important to deny the slave from re-registering.
- LOG(WARNING) << "Agent " << slaveInfo.id() << " at " << from
- << " (" << slaveInfo.hostname() << ") attempted to "
- << "re-register after removal; shutting it down";
-
- ShutdownMessage message;
- message.set_message("Agent attempted to re-register after removal");
- send(from, message);
- return;
- }
-
Slave* slave = slaves.registered.get(slaveInfo.id());
if (slave != nullptr) {
@@ -5126,10 +5105,13 @@ void Master::reregisterSlave(
slaves.reregistering.insert(slaveInfo.id());
- // This handles the case when the slave tries to re-register with
- // a failed over master, in which case we must consult the
- // registrar.
- registrar->apply(Owned<Operation>(new ReadmitSlave(slaveInfo)))
+ // Consult the registry to determine whether to readmit the
+ // slave. In the common case, the slave has been marked unreachable
+ // by the master, so we move the slave to the reachable list and
+ // readmit it. If the slave isn't in the unreachable list (which
+ // might occur if the slave's entry in the unreachable list is
+ // GC'd), we admit the slave anyway.
+ registrar->apply(Owned<Operation>(new MarkSlaveReachable(slaveInfo)))
.onAny(defer(self(),
&Self::_reregisterSlave,
slaveInfo,
@@ -5158,60 +5140,85 @@ void Master::_reregisterSlave(
CHECK(slaves.reregistering.contains(slaveInfo.id()));
slaves.reregistering.erase(slaveInfo.id());
- CHECK(!readmit.isDiscarded());
-
if (readmit.isFailed()) {
LOG(FATAL) << "Failed to readmit agent " << slaveInfo.id() << " at " << pid
<< " (" << slaveInfo.hostname() << "): " << readmit.failure();
- } else if (!readmit.get()) {
- LOG(WARNING) << "The agent " << slaveInfo.id() << " at "
- << pid << " (" << slaveInfo.hostname() << ") could not be"
- << " readmitted; shutting it down";
- slaves.removed.put(slaveInfo.id(), Nothing());
+ }
- ShutdownMessage message;
- message.set_message(
- "Agent attempted to re-register with unknown agent id " +
- stringify(slaveInfo.id()));
- send(pid, message);
- } else {
- // Re-admission succeeded.
- MachineID machineId;
- machineId.set_hostname(slaveInfo.hostname());
- machineId.set_ip(stringify(pid.address.ip));
+ CHECK(!readmit.isDiscarded());
- Slave* slave = new Slave(
- this,
- slaveInfo,
- pid,
- machineId,
- version,
- Clock::now(),
- checkpointedResources,
- executorInfos,
- tasks);
+ // `MarkSlaveReachable` registry operation should never fail.
+ CHECK(readmit.get());
- slave->reregisteredTime = Clock::now();
+ // Re-admission succeeded.
+ MachineID machineId;
+ machineId.set_hostname(slaveInfo.hostname());
+ machineId.set_ip(stringify(pid.address.ip));
- ++metrics->slave_reregistrations;
+ Slave* slave = new Slave(
+ this,
+ slaveInfo,
+ pid,
+ machineId,
+ version,
+ Clock::now(),
+ checkpointedResources,
+ executorInfos,
+ tasks);
- addSlave(slave, completedFrameworks);
+ slave->reregisteredTime = Clock::now();
- Duration pingTimeout =
- flags.agent_ping_timeout * flags.max_agent_ping_timeouts;
- MasterSlaveConnection connection;
- connection.set_total_ping_timeout_seconds(pingTimeout.secs());
+ ++metrics->slave_reregistrations;
- SlaveReregisteredMessage message;
- message.mutable_slave_id()->CopyFrom(slave->id);
- message.mutable_connection()->CopyFrom(connection);
- send(slave->pid, message);
+ // Check whether this master was the one that removed the
+ // reregistering agent from the cluster originally. This is false
+ // if the master has failed over since the agent was removed, for
+ // example.
+ bool slaveWasRemoved = slaves.removed.get(slave->id).isSome();
- LOG(INFO) << "Re-registered agent " << *slave
- << " with " << slave->info.resources();
+ addSlave(slave, completedFrameworks);
- __reregisterSlave(slave, tasks, frameworks);
+ Duration pingTimeout =
+ flags.agent_ping_timeout * flags.max_agent_ping_timeouts;
+ MasterSlaveConnection connection;
+ connection.set_total_ping_timeout_seconds(pingTimeout.secs());
+
+ SlaveReregisteredMessage message;
+ message.mutable_slave_id()->CopyFrom(slave->id);
+ message.mutable_connection()->CopyFrom(connection);
+ send(slave->pid, message);
+
+ LOG(INFO) << "Re-registered agent " << *slave
+ << " with " << slave->info.resources();
+
+ // Shutdown any frameworks running on the slave that don't have the
+ // PARTITION_AWARE capability, provided that this instance of the
+ // master previously added the slave to the `slaves.removed`
+ // collection. This matches the Mesos 1.0 "non-strict" semantics for
+ // frameworks that are not partition-aware: when a previously
+ // unreachable slave reregisters, its tasks are only shutdown if the
+ // master has not failed over.
+ if (slaveWasRemoved) {
+ foreach (const FrameworkInfo& framework, frameworks) {
+ if (!protobuf::frameworkHasCapability(
+ framework, FrameworkInfo::Capability::PARTITION_AWARE)) {
+ LOG(INFO) << "Shutting down framework " << framework.id()
+ << " at reregistered agent " << *slave
+ << " because the framework is not partition-aware";
+
+ ShutdownFrameworkMessage message;
+ message.mutable_framework_id()->MergeFrom(framework.id());
+ send(slave->pid, message);
+
+ // Remove the framework's tasks from the master's in-memory state.
+ foreachvalue (Task* task, utils::copy(slave->tasks[framework.id()])) {
+ removeTask(task);
+ }
+ }
+ }
}
+
+ __reregisterSlave(slave, tasks, frameworks);
}
@@ -5314,17 +5321,13 @@ void Master::updateSlave(
++metrics->messages_update_slave;
if (slaves.removed.get(slaveId).isSome()) {
- // If the slave is removed, we have already informed
- // frameworks that its tasks were LOST, so the slave should
- // shut down.
+ // If the slave has been removed, drop the status update. The
+ // master is no longer trying to health check this slave; when the
+ // slave realizes it hasn't received any pings from the master, it
+ // will eventually try to reregister.
LOG(WARNING)
<< "Ignoring update of agent with total oversubscribed resources "
- << oversubscribedResources << " on removed agent " << slaveId
- << " ; asking agent to shutdown";
-
- ShutdownMessage message;
- message.set_message("Update agent message from unknown agent");
- reply(message);
+ << oversubscribedResources << " on removed agent " << slaveId;
return;
}
@@ -5446,18 +5449,13 @@ void Master::statusUpdate(StatusUpdate update, const UPID& pid)
++metrics->messages_status_update;
if (slaves.removed.get(update.slave_id()).isSome()) {
- // If the slave is removed, we have already informed
- // frameworks that its tasks were LOST, so the slave should
- // shut down.
+ // If the slave has been removed, drop the status update. The
+ // master is no longer trying to health check this slave; when the
+ // slave realizes it hasn't received any pings from the master, it
+ // will eventually try to reregister.
LOG(WARNING) << "Ignoring status update " << update
<< " from removed agent " << pid
- << " with id " << update.slave_id() << " ; asking agent "
- << " to shutdown";
-
- ShutdownMessage message;
- message.set_message("Status update from unknown agent");
- send(pid, message);
-
+ << " with id " << update.slave_id();
metrics->invalid_status_updates++;
return;
}
@@ -5566,17 +5564,13 @@ void Master::exitedExecutor(
++metrics->messages_exited_executor;
if (slaves.removed.get(slaveId).isSome()) {
- // If the slave is removed, we have already informed
- // frameworks that its tasks were LOST, so the slave should
- // shut down.
+ // If the slave has been removed, drop the executor message. The
+ // master is no longer trying to health check this slave; when the
+ // slave realizes it hasn't received any pings from the master, it
+ // will eventually try to reregister.
LOG(WARNING) << "Ignoring exited executor '" << executorId
<< "' of framework " << frameworkId
- << " on removed agent " << slaveId
- << " ; asking agent to shutdown";
-
- ShutdownMessage message;
- message.set_message("Executor exited message from unknown agent");
- reply(message);
+ << " on removed agent " << slaveId;
return;
}
@@ -5653,27 +5647,182 @@ void Master::shutdown(
}
-void Master::shutdownSlave(const SlaveID& slaveId, const string& message)
+// TODO(neilc): Refactor to reduce code duplication with
+// `Master::removeSlave`.
+void Master::markUnreachable(const SlaveID& slaveId)
{
- if (!slaves.registered.contains(slaveId)) {
- // Possible when the SlaveObserver dispatches a message to
- // shutdown a slave but the slave is concurrently removed for
- // another reason (e.g., `UnregisterSlaveMessage` is received).
- LOG(WARNING) << "Unable to shutdown unknown agent " << slaveId;
+ Slave* slave = slaves.registered.get(slaveId);
+
+ if (slave == nullptr) {
+ // Possible when the `SlaveObserver` dispatches a message to mark an
+ // unhealthy slave as unreachable, but the slave is concurrently
+ // removed for another reason (e.g., `UnregisterSlaveMessage` is
+ // received).
+ LOG(WARNING) << "Unable to mark unknown agent "
+ << slaveId << " unreachable";
return;
}
- Slave* slave = slaves.registered.get(slaveId);
- CHECK_NOTNULL(slave);
+ LOG(INFO) << "Marking agent " << *slave
+ << " unreachable: health check timed out";
- LOG(WARNING) << "Shutting down agent " << *slave << " with message '"
- << message << "'";
+ // We want to remove the slave first, to avoid the allocator
+ // re-allocating the recovered resources.
+ //
+ // NOTE: Removing the slave is not sufficient for recovering the
+ // resources in the allocator, because the "Sorters" are updated
+ // only within recoverResources() (see MESOS-621). The calls to
+ // recoverResources() below are therefore required, even though
+ // the slave is already removed.
+ allocator->removeSlave(slave->id);
- ShutdownMessage message_;
- message_.set_message(message);
- send(slave->pid, message_);
+ // Transition the tasks to TASK_LOST and remove them, BUT do not
+ // send updates yet. Rather, build up the updates so that we can can
+ // send them after the slave has been moved to the unreachable list
+ // in the registry.
+ // TODO(neilc): Update this to send TASK_UNREACHABLE for
+ // partition-aware frameworks.
+ vector<StatusUpdate> updates;
+ foreachkey (const FrameworkID& frameworkId, utils::copy(slave->tasks)) {
+ Framework* framework = getFramework(frameworkId);
+ CHECK_NOTNULL(framework);
+
+ foreachvalue (Task* task, utils::copy(slave->tasks[frameworkId])) {
+ const StatusUpdate& update = protobuf::createStatusUpdate(
+ task->framework_id(),
+ task->slave_id(),
+ task->task_id(),
+ TASK_LOST,
+ TaskStatus::SOURCE_MASTER,
+ None(),
+ "Slave " + slave->info.hostname() + " is unreachable",
+ TaskStatus::REASON_SLAVE_REMOVED,
+ (task->has_executor_id() ?
+ Option<ExecutorID>(task->executor_id()) : None()));
+
+ updateTask(task, update);
+ removeTask(task);
+
+ updates.push_back(update);
+ }
+ }
+
+ // Remove executors from the slave for proper resource accounting.
+ foreachkey (const FrameworkID& frameworkId, utils::copy(slave->executors)) {
+ foreachkey (const ExecutorID& executorId,
+ utils::copy(slave->executors[frameworkId])) {
+ removeExecutor(slave, frameworkId, executorId);
+ }
+ }
+
+ foreach (Offer* offer, utils::copy(slave->offers)) {
+ // TODO(vinod): We don't need to call 'Allocator::recoverResources'
+ // once MESOS-621 is fixed.
+ allocator->recoverResources(
+ offer->framework_id(), slave->id, offer->resources(), None());
+
+ // Remove and rescind offers.
+ removeOffer(offer, true); // Rescind!
+ }
+
+ // Remove inverse offers because sending them for a slave that is
+ // unreachable doesn't make sense.
+ foreach (InverseOffer* inverseOffer, utils::copy(slave->inverseOffers)) {
+ // We don't need to update the allocator because we've already called
+ // `RemoveSlave()`.
+ // Remove and rescind inverse offers.
+ removeInverseOffer(inverseOffer, true); // Rescind!
+ }
+
+ // Mark the slave as being unreachable.
+ slaves.registered.remove(slave);
+ slaves.removed.put(slave->id, Nothing());
+ slaves.markingUnreachable.insert(slave->id);
+ authenticated.erase(slave->pid);
+
+ // Remove the slave from the `machines` mapping.
+ CHECK(machines.contains(slave->machineId));
+ CHECK(machines[slave->machineId].slaves.contains(slave->id));
+ machines[slave->machineId].slaves.erase(slave->id);
+
+ // Kill the slave observer.
+ terminate(slave->observer);
+ wait(slave->observer);
+ delete slave->observer;
+
+ // TODO(benh): unlink(slave->pid);
- removeSlave(slave, message, metrics->slave_removals_reason_unhealthy);
+ // Update the registry to move this slave from the list of admitted
+ // slaves to the list of unreachable slaves. Once this is completed,
+ // we can forward the TASK_LOST updates to the frameworks.
+ registrar->apply(Owned<Operation>(new MarkSlaveUnreachable(slave->info)))
+ .onAny(defer(self(),
+ &Self::_markUnreachable,
+ slave->info,
+ updates,
+ lambda::_1,
+ "health check timed out"));
+
+ delete slave;
+}
+
+
+void Master::_markUnreachable(
+ const SlaveInfo& slaveInfo,
+ const vector<StatusUpdate>& updates,
+ const Future<bool>& registrarResult,
+ const string& unreachableCause)
+{
+ CHECK(slaves.markingUnreachable.contains(slaveInfo.id()));
+ slaves.markingUnreachable.erase(slaveInfo.id());
+
+ if (registrarResult.isFailed()) {
+ LOG(FATAL) << "Failed to mark agent " << slaveInfo.id()
+ << " (" << slaveInfo.hostname() << ")"
+ << " unreachable in the registry: "
+ << registrarResult.failure();
+ }
+
+ CHECK(!registrarResult.isDiscarded());
+
+ // `MarkSlaveUnreachable` registry operation should never fail.
+ CHECK(registrarResult.get());
+
+ LOG(INFO) << "Marked agent " << slaveInfo.id() << " ("
+ << slaveInfo.hostname() << ") unreachable: "
+ << unreachableCause;
+
+ // TODO(neilc): Consider renaming these metrics or adding new
+ // metrics for the new PARTITION_AWARE semantics.
+ ++metrics->slave_removals;
+ ++metrics->slave_removals_reason_unhealthy;
+
+ // Forward the TASK_LOST updates on to the frameworks.
+ foreach (const StatusUpdate& update, updates) {
+ Framework* framework = getFramework(update.framework_id());
+
+ if (framework == nullptr) {
+ LOG(WARNING) << "Dropping update " << update << " from unknown framework "
+ << update.framework_id();
+ } else {
+ forward(update, UPID(), framework);
+ }
+ }
+
+ // Notify all frameworks of the lost slave.
+ foreachvalue (Framework* framework, frameworks.registered) {
+ LOG(INFO) << "Notifying framework " << *framework << " of lost agent "
+ << slaveInfo.id() << " (" << slaveInfo.hostname() << ") "
+ << "after recovering";
+ LostSlaveMessage message;
+ message.mutable_slave_id()->MergeFrom(slaveInfo.id());
+ framework->send(message);
+ }
+
+ // Finally, notify the `SlaveLost` hooks.
+ if (HookManager::hooksAvailable()) {
+ HookManager::masterSlaveLostHook(slaveInfo);
+ }
}
@@ -7057,27 +7206,27 @@ void Master::removeSlave(
void Master::_removeSlave(
const SlaveInfo& slaveInfo,
const vector<StatusUpdate>& updates,
- const Future<bool>& removed,
- const string& message,
+ const Future<bool>& registrarResult,
+ const string& removalCause,
Option<Counter> reason)
{
CHECK(slaves.removing.contains(slaveInfo.id()));
slaves.removing.erase(slaveInfo.id());
- CHECK(!removed.isDiscarded());
+ CHECK(!registrarResult.isDiscarded());
- if (removed.isFailed()) {
+ if (registrarResult.isFailed()) {
LOG(FATAL) << "Failed to remove agent " << slaveInfo.id()
<< " (" << slaveInfo.hostname() << ")"
- << " from the registrar: " << removed.failure();
+ << " from the registrar: " << registrarResult.failure();
}
- CHECK(removed.get())
+ CHECK(registrarResult.get())
<< "Agent " << slaveInfo.id() << " (" << slaveInfo.hostname() << ") "
<< "already removed from the registrar";
LOG(INFO) << "Removed agent " << slaveInfo.id() << " ("
- << slaveInfo.hostname() << "): " << message;
+ << slaveInfo.hostname() << "): " << removalCause;
++metrics->slave_removals;
http://git-wip-us.apache.org/repos/asf/mesos/blob/937c85f2/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index 4992ab0..b092de4 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -522,9 +522,7 @@ public:
const MachineID& machineId,
const Option<Unavailability>& unavailability);
- void shutdownSlave(
- const SlaveID& slaveId,
- const std::string& message);
+ void markUnreachable(const SlaveID& slaveId);
void authenticate(
const process::UPID& from,
@@ -686,9 +684,15 @@ protected:
const std::vector<Archive::Framework>& completedFrameworks =
std::vector<Archive::Framework>());
- // Remove the slave from the registrar. Called when the slave
+ void _markUnreachable(
+ const SlaveInfo& slaveInfo,
+ const std::vector<StatusUpdate>& updates,
+ const process::Future<bool>& registrarResult,
+ const std::string& unreachableCause);
+
+ // Mark a slave as unreachable in the registry. Called when the slave
// does not re-register in time after a master failover.
- Nothing removeSlave(const Registry::Slave& slave);
+ Nothing markUnreachableAfterFailover(const Registry::Slave& slave);
// Remove the slave from the registrar and from the master's state.
//
@@ -701,8 +705,8 @@ protected:
void _removeSlave(
const SlaveInfo& slaveInfo,
const std::vector<StatusUpdate>& updates,
- const process::Future<bool>& removed,
- const std::string& message,
+ const process::Future<bool>& registrarResult,
+ const std::string& removalCause,
Option<process::metrics::Counter> reason = None());
// Validates that the framework is authenticated, if required.
@@ -1684,12 +1688,12 @@ private:
// from the registry.
hashset<SlaveID> removing;
- // We track removed slaves to preserve the consistency
- // semantics of the pre-registrar code when a non-strict registrar
- // is being used. That is, if we remove a slave, we must make
- // an effort to prevent it from (re-)registering, sending updates,
- // etc. We keep a cache here to prevent this from growing in an
- // unbounded manner.
+ // Slaves that are in the process of being marked unreachable.
+ hashset<SlaveID> markingUnreachable;
+
+ // This collection includes agents that have gracefully shutdown,
+ // as well as those that have been marked unreachable. We keep a
+ // cache here to prevent this from growing in an unbounded manner.
// TODO(bmahler): Ideally we could use a cache with set semantics.
Cache<SlaveID, Nothing> removed;
@@ -2044,40 +2048,6 @@ private:
};
-// Implementation of slave readmission Registrar operation.
-class ReadmitSlave : public Operation
-{
-public:
- explicit ReadmitSlave(const SlaveInfo& _info) : info(_info)
- {
- CHECK(info.has_id()) << "SlaveInfo is missing the 'id' field";
- }
-
-protected:
- virtual Try<bool> perform(
- Registry* registry,
- hashset<SlaveID>* slaveIDs,
- bool strict)
- {
- if (slaveIDs->contains(info.id())) {
- return false; // No mutation.
- }
-
- if (strict) {
- return Error("Agent not yet admitted");
- } else {
- Registry::Slave* slave = registry->mutable_slaves()->add_slaves();
- slave->mutable_info()->CopyFrom(info);
- slaveIDs->insert(info.id());
- return true; // Mutation.
- }
- }
-
-private:
- const SlaveInfo info;
-};
-
-
// Implementation of slave removal Registrar operation.
class RemoveSlave : public Operation
{
http://git-wip-us.apache.org/repos/asf/mesos/blob/937c85f2/src/tests/master_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/master_tests.cpp b/src/tests/master_tests.cpp
index 6cde15f..514c1d9 100644
--- a/src/tests/master_tests.cpp
+++ b/src/tests/master_tests.cpp
@@ -1761,10 +1761,10 @@ TEST_F(MasterTest, SlavesEndpointTwoSlaves)
// This test ensures that when a slave is recovered from the registry
-// but does not re-register with the master, it is removed from the
-// registry and the framework is informed that the slave is lost, and
-// the slave is refused re-registration.
-TEST_F(MasterTest, RecoveredSlaveDoesNotReregister)
+// but does not re-register with the master, it is marked unreachable
+// in the registry, the framework is informed that the slave is lost,
+// and the slave is allowed to re-register.
+TEST_F(MasterTest, RecoveredSlaveCanReregister)
{
// Step 1: Start a master.
master::Flags masterFlags = CreateMasterFlags();
@@ -1826,15 +1826,15 @@ TEST_F(MasterTest, RecoveredSlaveDoesNotReregister)
Clock::resume();
- // Step 7: Ensure the slave cannot re-register!
- Future<ShutdownMessage> shutdownMessage =
- FUTURE_PROTOBUF(ShutdownMessage(), master.get()->pid, _);
+ // Step 7: Ensure the slave can re-register.
+ Future<SlaveReregisteredMessage> slaveReregisteredMessage =
+ FUTURE_PROTOBUF(SlaveReregisteredMessage(), master.get()->pid, _);
detector = master.get()->createDetector();
slave = StartSlave(detector.get(), slaveFlags);
ASSERT_SOME(slave);
- AWAIT_READY(shutdownMessage);
+ AWAIT_READY(slaveReregisteredMessage);
driver.stop();
driver.join();
http://git-wip-us.apache.org/repos/asf/mesos/blob/937c85f2/src/tests/partition_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/partition_tests.cpp b/src/tests/partition_tests.cpp
index 8e3827a..34622b3 100644
--- a/src/tests/partition_tests.cpp
+++ b/src/tests/partition_tests.cpp
@@ -21,6 +21,7 @@
#include <process/clock.hpp>
#include <process/future.hpp>
#include <process/gmock.hpp>
+#include <process/http.hpp>
#include <process/owned.hpp>
#include <process/pid.hpp>
@@ -58,6 +59,9 @@ using process::Message;
using process::Owned;
using process::PID;
+using process::http::OK;
+using process::http::Response;
+
using std::vector;
using testing::_;
@@ -160,17 +164,13 @@ TEST_P(PartitionTest, PartitionedSlave)
}
-// The purpose of this test is to ensure that when slaves are removed
-// from the master, and then attempt to re-register, we deny the
-// re-registration by sending a ShutdownMessage to the slave.
-// Why? Because during a network partition, the master will remove a
-// partitioned slave, thus sending its tasks to LOST. At this point,
-// when the partition is removed, the slave will attempt to
-// re-register with its running tasks. We've already notified
-// frameworks that these tasks were LOST, so we have to have the
-// slave shut down.
-TEST_P(PartitionTest, PartitionedSlaveReregistration)
+// This test checks that a slave can reregister with the master after
+// a partition, and that PARTITION_AWARE tasks running on the slave
+// continue to run.
+TEST_P(PartitionTest, ReregisterSlavePartitionAware)
{
+ Clock::pause();
+
master::Flags masterFlags = CreateMasterFlags();
masterFlags.registry_strict = GetParam();
@@ -186,47 +186,37 @@ TEST_P(PartitionTest, PartitionedSlaveReregistration)
DROP_PROTOBUFS(PongSlaveMessage(), _, _);
- MockExecutor exec(DEFAULT_EXECUTOR_ID);
- TestContainerizer containerizer(&exec);
-
StandaloneMasterDetector detector(master.get()->pid);
- Try<Owned<cluster::Slave>> slave = StartSlave(&detector, &containerizer);
+ Try<Owned<cluster::Slave>> slave = StartSlave(&detector);
ASSERT_SOME(slave);
+ // Start a scheduler. The scheduler has the PARTITION_AWARE
+ // capability, so we expect its tasks to continue running when the
+ // partitioned agent reregisters.
+ FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
+ frameworkInfo.add_capabilities()->set_type(
+ FrameworkInfo::Capability::PARTITION_AWARE);
+
MockScheduler sched;
MesosSchedulerDriver driver(
- &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+ &sched, frameworkInfo, master.get()->pid, DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _));
Future<vector<Offer>> offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(FutureArg<1>(&offers))
- .WillRepeatedly(Return());
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(offers);
- ASSERT_NE(0u, offers.get().size());
+ ASSERT_FALSE(offers.get().empty());
- // Launch a task. This is to ensure the task is killed by the slave,
- // during shutdown.
- TaskID taskId;
- taskId.set_value("1");
+ Offer offer = offers.get()[0];
- TaskInfo task;
- task.set_name("");
- task.mutable_task_id()->MergeFrom(taskId);
- task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id());
- task.mutable_resources()->MergeFrom(offers.get()[0].resources());
- task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO);
- task.mutable_executor()->mutable_command()->set_value("sleep 60");
-
- // Set up the expectations for launching the task.
- EXPECT_CALL(exec, registered(_, _, _, _));
- EXPECT_CALL(exec, launchTask(_, _))
- .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING));
+ TaskInfo task = createTask(offer, "sleep 60");
Future<TaskStatus> runningStatus;
EXPECT_CALL(sched, statusUpdate(&driver, _))
@@ -235,33 +225,171 @@ TEST_P(PartitionTest, PartitionedSlaveReregistration)
Future<Nothing> statusUpdateAck = FUTURE_DISPATCH(
slave.get()->pid, &Slave::_statusUpdateAcknowledgement);
- driver.launchTasks(offers.get()[0].id(), {task});
+ driver.launchTasks(offer.id(), {task});
AWAIT_READY(runningStatus);
EXPECT_EQ(TASK_RUNNING, runningStatus.get().state());
+ EXPECT_EQ(task.task_id(), runningStatus.get().task_id());
- // Wait for the slave to have handled the acknowledgment prior
- // to pausing the clock.
- AWAIT_READY(statusUpdateAck);
+ const SlaveID slaveId = runningStatus.get().slave_id();
- // Drop the first shutdown message from the master (simulated
- // partition), allow the second shutdown message to pass when
- // the slave re-registers.
- Future<ShutdownMessage> shutdownMessage =
- DROP_PROTOBUF(ShutdownMessage(), _, slave.get()->pid);
+ AWAIT_READY(statusUpdateAck);
- Future<TaskStatus> lostStatus;
+ // Now, induce a partition of the slave by having the master
+ // timeout the slave.
+ Future<TaskStatus> unreachableStatus;
EXPECT_CALL(sched, statusUpdate(&driver, _))
- .WillOnce(FutureArg<1>(&lostStatus));
+ .WillOnce(FutureArg<1>(&unreachableStatus));
Future<Nothing> slaveLost;
EXPECT_CALL(sched, slaveLost(&driver, _))
.WillOnce(FutureSatisfy(&slaveLost));
+ size_t pings = 0;
+ while (true) {
+ AWAIT_READY(ping);
+ pings++;
+ if (pings == masterFlags.max_agent_ping_timeouts) {
+ break;
+ }
+ ping = FUTURE_MESSAGE(Eq(PingSlaveMessage().GetTypeName()), _, _);
+ Clock::advance(masterFlags.agent_ping_timeout);
+ }
+
+ Clock::advance(masterFlags.agent_ping_timeout);
+
+ // TODO(neilc): Update this when TASK_UNREACHABLE is introduced.
+ AWAIT_READY(unreachableStatus);
+ EXPECT_EQ(TASK_LOST, unreachableStatus.get().state());
+ EXPECT_EQ(TaskStatus::REASON_SLAVE_REMOVED, unreachableStatus.get().reason());
+ EXPECT_EQ(task.task_id(), unreachableStatus.get().task_id());
+ EXPECT_EQ(slaveId, unreachableStatus.get().slave_id());
+
+ AWAIT_READY(slaveLost);
+
+ JSON::Object stats = Metrics();
+ EXPECT_EQ(1, stats.values["master/tasks_lost"]);
+ EXPECT_EQ(0, stats.values["master/tasks_unreachable"]);
+ EXPECT_EQ(1, stats.values["master/slave_unreachable_scheduled"]);
+ EXPECT_EQ(1, stats.values["master/slave_unreachable_completed"]);
+ EXPECT_EQ(1, stats.values["master/slave_removals"]);
+ EXPECT_EQ(1, stats.values["master/slave_removals/reason_unhealthy"]);
+
+ // We now complete the partition on the slave side as well. We
+ // simulate a master loss event, which would normally happen during
+ // a network partition. The slave should then reregister with the
+ // master.
+ detector.appoint(None());
+
+ Future<SlaveReregisteredMessage> slaveReregistered = FUTURE_PROTOBUF(
+ SlaveReregisteredMessage(), master.get()->pid, slave.get()->pid);
+
+ detector.appoint(master.get()->pid);
+
+ AWAIT_READY(slaveReregistered);
+
+ // Perform explicit reconciliation; the task should still be running.
+ TaskStatus status;
+ status.mutable_task_id()->CopyFrom(task.task_id());
+ status.mutable_slave_id()->CopyFrom(slaveId);
+ status.set_state(TASK_STAGING); // Dummy value.
+
+ Future<TaskStatus> reconcileUpdate;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&reconcileUpdate));
+
+ driver.reconcileTasks({status});
+
+ AWAIT_READY(reconcileUpdate);
+ EXPECT_EQ(TASK_RUNNING, reconcileUpdate.get().state());
+ EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate.get().reason());
+
+ Clock::resume();
+
+ driver.stop();
+ driver.join();
+}
+
+
+// This test checks that a slave can reregister with the master after
+// a partition, and that non-PARTITION_AWARE tasks running on the
+// slave are shutdown.
+TEST_P(PartitionTest, ReregisterSlaveNotPartitionAware)
+{
Clock::pause();
+ master::Flags masterFlags = CreateMasterFlags();
+ masterFlags.registry_strict = GetParam();
+
+ Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
+ ASSERT_SOME(master);
+
+ // Allow the master to PING the slave, but drop all PONG messages
+ // from the slave. Note that we don't match on the master / slave
+ // PIDs because it's actually the SlaveObserver Process that sends
+ // the pings.
+ Future<Message> ping = FUTURE_MESSAGE(
+ Eq(PingSlaveMessage().GetTypeName()), _, _);
+
+ DROP_PROTOBUFS(PongSlaveMessage(), _, _);
+
+ StandaloneMasterDetector detector(master.get()->pid);
+
+ Try<Owned<cluster::Slave>> slave = StartSlave(&detector);
+ ASSERT_SOME(slave);
+
+ // Start a scheduler. The scheduler is not PARTITION_AWARE, so we
+ // expect its tasks to be shutdown when the partitioned agent
+ // reregisters.
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(&driver, _, _));
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(offers);
+ ASSERT_FALSE(offers.get().empty());
+
+ Offer offer = offers.get()[0];
+
+ TaskInfo task = createTask(offer, "sleep 60");
+
+ Future<TaskStatus> runningStatus;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&runningStatus));
+
+ Future<Nothing> statusUpdateAck = FUTURE_DISPATCH(
+ slave.get()->pid, &Slave::_statusUpdateAcknowledgement);
+
+ driver.launchTasks(offer.id(), {task});
+
+ AWAIT_READY(runningStatus);
+ EXPECT_EQ(TASK_RUNNING, runningStatus.get().state());
+ EXPECT_EQ(task.task_id(), runningStatus.get().task_id());
+
+ const SlaveID slaveId = runningStatus.get().slave_id();
+
+ AWAIT_READY(statusUpdateAck);
+
// Now, induce a partition of the slave by having the master
// timeout the slave.
+ Future<TaskStatus> lostStatus;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&lostStatus));
+
+ // Note that we expect to get `slaveLost` callbacks in both
+ // schedulers, regardless of PARTITION_AWARE.
+ Future<Nothing> slaveLost;
+ EXPECT_CALL(sched, slaveLost(&driver, _))
+ .WillOnce(FutureSatisfy(&slaveLost));
+
size_t pings = 0;
while (true) {
AWAIT_READY(ping);
@@ -275,71 +403,82 @@ TEST_P(PartitionTest, PartitionedSlaveReregistration)
Clock::advance(masterFlags.agent_ping_timeout);
- // The master will have notified the framework of the lost task.
+ // The scheduler should see TASK_LOST because it is not
+ // PARTITION_AWARE.
AWAIT_READY(lostStatus);
EXPECT_EQ(TASK_LOST, lostStatus.get().state());
EXPECT_EQ(TaskStatus::REASON_SLAVE_REMOVED, lostStatus.get().reason());
+ EXPECT_EQ(task.task_id(), lostStatus.get().task_id());
+ EXPECT_EQ(slaveId, lostStatus.get().slave_id());
- // Wait for the master to attempt to shut down the slave.
- AWAIT_READY(shutdownMessage);
-
- // The master will notify the framework that the slave was lost.
AWAIT_READY(slaveLost);
JSON::Object stats = Metrics();
EXPECT_EQ(1, stats.values["master/tasks_lost"]);
+ EXPECT_EQ(0, stats.values["master/tasks_unreachable"]);
+ EXPECT_EQ(1, stats.values["master/slave_unreachable_scheduled"]);
+ EXPECT_EQ(1, stats.values["master/slave_unreachable_completed"]);
EXPECT_EQ(1, stats.values["master/slave_removals"]);
EXPECT_EQ(1, stats.values["master/slave_removals/reason_unhealthy"]);
- Clock::resume();
-
- // We now complete the partition on the slave side as well. This
- // is done by simulating a master loss event which would normally
- // occur during a network partition.
+ // We now complete the partition on the slave side as well. We
+ // simulate a master loss event, which would normally happen during
+ // a network partition. The slave should then reregister with the
+ // master.
detector.appoint(None());
- Future<Nothing> shutdown;
- EXPECT_CALL(exec, shutdown(_))
- .WillOnce(FutureSatisfy(&shutdown));
-
- shutdownMessage = FUTURE_PROTOBUF(ShutdownMessage(), _, slave.get()->pid);
+ Future<SlaveReregisteredMessage> slaveReregistered = FUTURE_PROTOBUF(
+ SlaveReregisteredMessage(), master.get()->pid, slave.get()->pid);
- // Have the slave re-register with the master.
detector.appoint(master.get()->pid);
- // Upon re-registration, the master will shutdown the slave.
- // The slave will then shut down the executor.
- AWAIT_READY(shutdownMessage);
- AWAIT_READY(shutdown);
+ AWAIT_READY(slaveReregistered);
+
+ // Perform explicit reconciliation. The task should not be running
+ // (TASK_LOST) because the framework is not PARTITION_AWARE.
+ TaskStatus status;
+ status.mutable_task_id()->CopyFrom(task.task_id());
+ status.mutable_slave_id()->CopyFrom(slaveId);
+ status.set_state(TASK_STAGING); // Dummy value.
+
+ Future<TaskStatus> reconcileUpdate;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&reconcileUpdate));
+
+ driver.reconcileTasks({status});
+
+ AWAIT_READY(reconcileUpdate);
+ EXPECT_EQ(TASK_LOST, reconcileUpdate.get().state());
+ EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate.get().reason());
+
+ Clock::resume();
driver.stop();
driver.join();
}
-// The purpose of this test is to ensure that when slaves are removed
-// from the master, and then attempt to send status updates, we send
-// a ShutdownMessage to the slave. Why? Because during a network
-// partition, the master will remove a partitioned slave, thus sending
-// its tasks to LOST. At this point, when the partition is removed,
-// the slave may attempt to send updates if it was unaware that the
-// master removed it. We've already notified frameworks that these
-// tasks were LOST, so we have to have the slave shut down.
+// This test checks how Mesos behaves when a slave is removed because
+// it fails health checks, and then the slave sends a status update
+// (because it does not realize that it is partitioned from the
+// master's POV). In prior Mesos versions, the master would shutdown
+// the slave in this situation. In Mesos >= 1.1, the master will drop
+// the status update; the slave will eventually try to reregister.
TEST_P(PartitionTest, PartitionedSlaveStatusUpdates)
{
+ Clock::pause();
+
master::Flags masterFlags = CreateMasterFlags();
masterFlags.registry_strict = GetParam();
Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
ASSERT_SOME(master);
- // Allow the master to PING the slave, but drop all PONG messages
- // from the slave. Note that we don't match on the master / slave
- // PIDs because it's actually the SlaveObserver Process that sends
- // the pings.
- Future<Message> ping = FUTURE_MESSAGE(
- Eq(PingSlaveMessage().GetTypeName()), _, _);
-
+ // Drop both PINGs from master to slave and PONGs from slave to
+ // master. Note that we don't match on the master / slave PIDs
+ // because it's actually the SlaveObserver Process that sends pings
+ // and receives pongs.
+ DROP_PROTOBUFS(PingSlaveMessage(), _, _);
DROP_PROTOBUFS(PongSlaveMessage(), _, _);
Future<SlaveRegisteredMessage> slaveRegisteredMessage =
@@ -370,12 +509,6 @@ TEST_P(PartitionTest, PartitionedSlaveStatusUpdates)
AWAIT_READY(frameworkId);
- // Drop the first shutdown message from the master (simulated
- // partition), allow the second shutdown message to pass when
- // the slave sends an update.
- Future<ShutdownMessage> shutdownMessage =
- DROP_PROTOBUF(ShutdownMessage(), _, slave.get()->pid);
-
EXPECT_CALL(sched, offerRescinded(&driver, _))
.WillRepeatedly(Return());
@@ -383,72 +516,107 @@ TEST_P(PartitionTest, PartitionedSlaveStatusUpdates)
EXPECT_CALL(sched, slaveLost(&driver, _))
.WillOnce(FutureSatisfy(&slaveLost));
- Clock::pause();
+ // Now, induce a partition of the slave by having the master timeout
+ // the slave. The master will remove the slave; the slave will also
+ // realize that it hasn't seen any pings from the master and try to
+ // reregister. We don't want to let the slave reregister yet, so we
+ // drop the first message in the reregistration protocol, which is
+ // AuthenticateMessage since agent auth is enabled.
+ Future<AuthenticateMessage> authenticateMessage =
+ DROP_PROTOBUF(AuthenticateMessage(), _, _);
- // Now, induce a partition of the slave by having the master
- // timeout the slave.
- size_t pings = 0;
- while (true) {
- AWAIT_READY(ping);
- pings++;
- if (pings == masterFlags.max_agent_ping_timeouts) {
- break;
- }
- ping = FUTURE_MESSAGE(Eq(PingSlaveMessage().GetTypeName()), _, _);
+ for (size_t i = 0; i < masterFlags.max_agent_ping_timeouts; i++) {
Clock::advance(masterFlags.agent_ping_timeout);
+ Clock::settle();
}
- Clock::advance(masterFlags.agent_ping_timeout);
-
- // Wait for the master to attempt to shut down the slave.
- AWAIT_READY(shutdownMessage);
-
// The master will notify the framework that the slave was lost.
AWAIT_READY(slaveLost);
+ // Slave will try to authenticate for reregistration; message dropped.
+ AWAIT_READY(authenticateMessage);
+
JSON::Object stats = Metrics();
+ EXPECT_EQ(1, stats.values["master/slave_unreachable_scheduled"]);
+ EXPECT_EQ(1, stats.values["master/slave_unreachable_completed"]);
EXPECT_EQ(1, stats.values["master/slave_removals"]);
EXPECT_EQ(1, stats.values["master/slave_removals/reason_unhealthy"]);
- shutdownMessage = FUTURE_PROTOBUF(ShutdownMessage(), _, slave.get()->pid);
-
// At this point, the slave still thinks it's registered, so we
// simulate a status update coming from the slave.
- TaskID taskId;
- taskId.set_value("task_id");
- const StatusUpdate& update = protobuf::createStatusUpdate(
+ TaskID taskId1;
+ taskId1.set_value("task_id1");
+
+ const StatusUpdate& update1 = protobuf::createStatusUpdate(
frameworkId.get(),
slaveId,
- taskId,
+ taskId1,
TASK_RUNNING,
TaskStatus::SOURCE_SLAVE,
UUID::random());
- StatusUpdateMessage message;
- message.mutable_update()->CopyFrom(update);
- message.set_pid(stringify(slave.get()->pid));
+ StatusUpdateMessage message1;
+ message1.mutable_update()->CopyFrom(update1);
+ message1.set_pid(stringify(slave.get()->pid));
+
+ // The scheduler should not receive the status update.
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .Times(0);
- process::post(master.get()->pid, message);
+ process::post(master.get()->pid, message1);
+ Clock::settle();
- // The master should shutdown the slave upon receiving the update.
- AWAIT_READY(shutdownMessage);
+ // Advance the clock so that the slaves notices that it still hasn't
+ // seen any pings from the master, which will cause it to try to
+ // reregister again. This time reregistration should succeed.
+ Future<SlaveReregisteredMessage> slaveReregistered = FUTURE_PROTOBUF(
+ SlaveReregisteredMessage(), master.get()->pid, slave.get()->pid);
- Clock::resume();
+ for (size_t i = 0; i < masterFlags.max_agent_ping_timeouts; i++) {
+ Clock::advance(masterFlags.agent_ping_timeout);
+ Clock::settle();
+ }
+
+ AWAIT_READY(slaveReregistered);
+
+ // Since the slave has reregistered, a status update from the slave
+ // should now be forwarded to the scheduler.
+ Future<StatusUpdateMessage> statusUpdate =
+ DROP_PROTOBUF(StatusUpdateMessage(), master.get()->pid, _);
+
+ TaskID taskId2;
+ taskId2.set_value("task_id2");
+
+ const StatusUpdate& update2 = protobuf::createStatusUpdate(
+ frameworkId.get(),
+ slaveId,
+ taskId2,
+ TASK_RUNNING,
+ TaskStatus::SOURCE_SLAVE,
+ UUID::random());
+
+ StatusUpdateMessage message2;
+ message2.mutable_update()->CopyFrom(update2);
+ message2.set_pid(stringify(slave.get()->pid));
+
+ process::post(master.get()->pid, message2);
+
+ AWAIT_READY(statusUpdate);
+ EXPECT_EQ(taskId2, statusUpdate->update().status().task_id());
driver.stop();
driver.join();
+
+ Clock::resume();
}
-// The purpose of this test is to ensure that when slaves are removed
-// from the master, and then attempt to send exited executor messages,
-// we send a ShutdownMessage to the slave. Why? Because during a
-// network partition, the master will remove a partitioned slave, thus
-// sending its tasks to LOST. At this point, when the partition is
-// removed, the slave may attempt to send exited executor messages if
-// it was unaware that the master removed it. We've already
-// notified frameworks that the tasks under the executors were LOST,
-// so we have to have the slave shut down.
+// This test checks how Mesos behaves when a slave is removed, and
+// then the slave sends an ExitedExecutorMessage (because it does not
+// realize it is partitioned from the master's POV). In prior Mesos
+// versions, the master would shutdown the slave in this situation. In
+// Mesos >= 1.1, the master will drop the message; the slave will
+// eventually try to reregister.
TEST_P(PartitionTest, PartitionedSlaveExitedExecutor)
{
master::Flags masterFlags = CreateMasterFlags();
@@ -494,16 +662,7 @@ TEST_P(PartitionTest, PartitionedSlaveExitedExecutor)
// Launch a task. This allows us to have the slave send an
// ExitedExecutorMessage.
- TaskID taskId;
- taskId.set_value("1");
-
- TaskInfo task;
- task.set_name("");
- task.mutable_task_id()->MergeFrom(taskId);
- task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id());
- task.mutable_resources()->MergeFrom(offers.get()[0].resources());
- task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO);
- task.mutable_executor()->mutable_command()->set_value("sleep 60");
+ TaskInfo task = createTask(offers.get()[0], "sleep 60", DEFAULT_EXECUTOR_ID);
// Set up the expectations for launching the task.
EXPECT_CALL(exec, registered(_, _, _, _));
@@ -511,19 +670,11 @@ TEST_P(PartitionTest, PartitionedSlaveExitedExecutor)
EXPECT_CALL(exec, launchTask(_, _))
.WillOnce(SendStatusUpdateFromTask(TASK_RUNNING));
- // Drop all the status updates from the slave, so that we can
- // ensure the ExitedExecutorMessage is what triggers the slave
- // shutdown.
+ // Drop all the status updates from the slave.
DROP_PROTOBUFS(StatusUpdateMessage(), _, master.get()->pid);
driver.launchTasks(offers.get()[0].id(), {task});
- // Drop the first shutdown message from the master (simulated
- // partition) and allow the second shutdown message to pass when
- // triggered by the ExitedExecutorMessage.
- Future<ShutdownMessage> shutdownMessage =
- DROP_PROTOBUF(ShutdownMessage(), _, slave.get()->pid);
-
Future<TaskStatus> lostStatus;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&lostStatus));
@@ -549,31 +700,34 @@ TEST_P(PartitionTest, PartitionedSlaveExitedExecutor)
Clock::advance(masterFlags.agent_ping_timeout);
- // The master will have notified the framework of the lost task.
+ // The master will notify the framework of the lost task.
AWAIT_READY(lostStatus);
EXPECT_EQ(TASK_LOST, lostStatus.get().state());
EXPECT_EQ(TaskStatus::REASON_SLAVE_REMOVED, lostStatus.get().reason());
- // Wait for the master to attempt to shut down the slave.
- AWAIT_READY(shutdownMessage);
-
// The master will notify the framework that the slave was lost.
AWAIT_READY(slaveLost);
JSON::Object stats = Metrics();
EXPECT_EQ(1, stats.values["master/tasks_lost"]);
+ EXPECT_EQ(1, stats.values["master/slave_unreachable_scheduled"]);
+ EXPECT_EQ(1, stats.values["master/slave_unreachable_completed"]);
EXPECT_EQ(1, stats.values["master/slave_removals"]);
EXPECT_EQ(1, stats.values["master/slave_removals/reason_unhealthy"]);
- shutdownMessage = FUTURE_PROTOBUF(ShutdownMessage(), _, slave.get()->pid);
+ EXPECT_CALL(sched, executorLost(&driver, _, _, _))
+ .Times(0);
// Induce an ExitedExecutorMessage from the slave.
- containerizer.destroy(
- frameworkId.get(), DEFAULT_EXECUTOR_INFO.executor_id());
-
- // Upon receiving the message, the master will shutdown the slave.
- AWAIT_READY(shutdownMessage);
-
+ containerizer.destroy(frameworkId.get(), DEFAULT_EXECUTOR_ID);
+
+ // The master will drop the ExitedExecutorMessage. We do not
+ // currently support reliable delivery of ExitedExecutorMessages, so
+ // the message will not be delivered if/when the slave reregisters.
+ //
+ // TODO(neilc): Update this test to check for reliable delivery once
+ // MESOS-4308 is fixed.
+ Clock::settle();
Clock::resume();
driver.stop();
[23/27] mesos git commit: Removed the "strict" flag from the registry
operation interface.
Posted by vi...@apache.org.
Removed the "strict" flag from the registry operation interface.
This flag is now ignored by all registry operations, so this should
not result in a functional change.
Review: https://reviews.apache.org/r/51954/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/ef9211f0
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/ef9211f0
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/ef9211f0
Branch: refs/heads/master
Commit: ef9211f0e1a77896f0c5cee032dc032e9d3a0c5c
Parents: 9fa9961
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:49:19 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:49:19 2016 -0700
----------------------------------------------------------------------
src/master/maintenance.cpp | 9 +++------
src/master/maintenance.hpp | 15 +++------------
src/master/master.hpp | 22 +++++-----------------
src/master/quota.cpp | 6 ++----
src/master/quota.hpp | 13 ++-----------
src/master/registrar.cpp | 7 ++-----
src/master/registrar.hpp | 18 +++---------------
src/master/weights.cpp | 4 +++-
src/master/weights.hpp | 5 +----
9 files changed, 24 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/ef9211f0/src/master/maintenance.cpp
----------------------------------------------------------------------
diff --git a/src/master/maintenance.cpp b/src/master/maintenance.cpp
index 7da61c7..c07b815 100644
--- a/src/master/maintenance.cpp
+++ b/src/master/maintenance.cpp
@@ -44,8 +44,7 @@ UpdateSchedule::UpdateSchedule(
Try<bool> UpdateSchedule::perform(
Registry* registry,
- hashset<SlaveID>* slaveIDs,
- bool strict)
+ hashset<SlaveID>* /*slaveIDs*/)
{
// Put the machines in the existing schedule into a set.
hashset<MachineID> existing;
@@ -123,8 +122,7 @@ StartMaintenance::StartMaintenance(
Try<bool> StartMaintenance::perform(
Registry* registry,
- hashset<SlaveID>* slaveIDs,
- bool strict)
+ hashset<SlaveID>* /*perform*/)
{
// Flip the mode of all targeted machines.
bool changed = false;
@@ -153,8 +151,7 @@ StopMaintenance::StopMaintenance(
Try<bool> StopMaintenance::perform(
Registry* registry,
- hashset<SlaveID>* slaveIDs,
- bool strict)
+ hashset<SlaveID>* /*slaveIDs*/)
{
// Delete the machine info entry of all targeted machines.
// i.e. Transition them into `UP` mode.
http://git-wip-us.apache.org/repos/asf/mesos/blob/ef9211f0/src/master/maintenance.hpp
----------------------------------------------------------------------
diff --git a/src/master/maintenance.hpp b/src/master/maintenance.hpp
index 87fbb2a..b1e176c 100644
--- a/src/master/maintenance.hpp
+++ b/src/master/maintenance.hpp
@@ -54,10 +54,7 @@ public:
const mesos::maintenance::Schedule& _schedule);
protected:
- Try<bool> perform(
- Registry* registry,
- hashset<SlaveID>* slaveIDs,
- bool strict);
+ Try<bool> perform(Registry* registry, hashset<SlaveID>* slaveIDs);
private:
const mesos::maintenance::Schedule schedule;
@@ -78,10 +75,7 @@ public:
const google::protobuf::RepeatedPtrField<MachineID>& _ids);
protected:
- Try<bool> perform(
- Registry* registry,
- hashset<SlaveID>* slaveIDs,
- bool strict);
+ Try<bool> perform(Registry* registry, hashset<SlaveID>* slaveIDs);
private:
hashset<MachineID> ids;
@@ -101,10 +95,7 @@ public:
const google::protobuf::RepeatedPtrField<MachineID>& _ids);
protected:
- Try<bool> perform(
- Registry* registry,
- hashset<SlaveID>* slaveIDs,
- bool strict);
+ Try<bool> perform(Registry* registry, hashset<SlaveID>* slaveIDs);
private:
hashset<MachineID> ids;
http://git-wip-us.apache.org/repos/asf/mesos/blob/ef9211f0/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index a051742..35db198 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -1929,10 +1929,7 @@ public:
}
protected:
- virtual Try<bool> perform(
- Registry* registry,
- hashset<SlaveID>* slaveIDs,
- bool)
+ virtual Try<bool> perform(Registry* registry, hashset<SlaveID>* slaveIDs)
{
// Check if this slave is currently admitted. This should only
// happen if there is a slaveID collision, but that is extremely
@@ -1964,10 +1961,7 @@ public:
}
protected:
- virtual Try<bool> perform(
- Registry* registry,
- hashset<SlaveID>* slaveIDs,
- bool)
+ virtual Try<bool> perform(Registry* registry, hashset<SlaveID>* slaveIDs)
{
// As currently implemented, this should not be possible: the
// master will only mark slaves unreachable that are currently
@@ -2017,10 +2011,7 @@ public:
}
protected:
- virtual Try<bool> perform(
- Registry* registry,
- hashset<SlaveID>* slaveIDs,
- bool)
+ virtual Try<bool> perform(Registry* registry, hashset<SlaveID>* slaveIDs)
{
// A slave might try to reregister that appears in the list of
// admitted slaves. This can occur when the master fails over:
@@ -2073,7 +2064,7 @@ public:
: toRemove(_toRemove) {}
protected:
- virtual Try<bool> perform(Registry* registry, hashset<SlaveID>*, bool)
+ virtual Try<bool> perform(Registry* registry, hashset<SlaveID>* /*slaveIDs*/)
{
// Attempt to remove the SlaveIDs in `toRemove` from the
// unreachable list. Some SlaveIDs in `toRemove` might not appear
@@ -2118,10 +2109,7 @@ public:
}
protected:
- virtual Try<bool> perform(
- Registry* registry,
- hashset<SlaveID>* slaveIDs,
- bool)
+ virtual Try<bool> perform(Registry* registry, hashset<SlaveID>* slaveIDs)
{
for (int i = 0; i < registry->slaves().slaves().size(); i++) {
const Registry::Slave& slave = registry->slaves().slaves(i);
http://git-wip-us.apache.org/repos/asf/mesos/blob/ef9211f0/src/master/quota.cpp
----------------------------------------------------------------------
diff --git a/src/master/quota.cpp b/src/master/quota.cpp
index bec6bd9..847ec06 100644
--- a/src/master/quota.cpp
+++ b/src/master/quota.cpp
@@ -43,8 +43,7 @@ UpdateQuota::UpdateQuota(const QuotaInfo& quotaInfo)
Try<bool> UpdateQuota::perform(
Registry* registry,
- hashset<SlaveID>*,
- bool)
+ hashset<SlaveID>* /*slaveIDs*/)
{
// If there is already quota stored for the role, update the entry.
foreach (Registry::Quota& quota, *registry->mutable_quotas()) {
@@ -66,8 +65,7 @@ RemoveQuota::RemoveQuota(const string& _role) : role(_role) {}
Try<bool> RemoveQuota::perform(
Registry* registry,
- hashset<SlaveID>*,
- bool)
+ hashset<SlaveID>* /*slaveIDs*/)
{
// Remove quota for the role if a corresponding entry exists.
for (int i = 0; i < registry->quotas().size(); ++i) {
http://git-wip-us.apache.org/repos/asf/mesos/blob/ef9211f0/src/master/quota.hpp
----------------------------------------------------------------------
diff --git a/src/master/quota.hpp b/src/master/quota.hpp
index 3cb194a..479112d 100644
--- a/src/master/quota.hpp
+++ b/src/master/quota.hpp
@@ -46,9 +46,6 @@ namespace quota {
// will render the operation hanging (i.e. `Future` for the operation
// will not be set).
-// The `strict` flag is not relevant for quota operations: they will
-// always succeed, even if the flag is set to `true`.
-
/**
* Sets quota for a role. No assumptions are made here: the role may
* be unknown to the master, or quota can be already set for the role.
@@ -67,10 +64,7 @@ public:
explicit UpdateQuota(const mesos::quota::QuotaInfo& quotaInfo);
protected:
- Try<bool> perform(
- Registry* registry,
- hashset<SlaveID>* slaveIDs,
- bool strict);
+ Try<bool> perform(Registry* registry, hashset<SlaveID>* slaveIDs);
private:
const mesos::quota::QuotaInfo info;
@@ -89,10 +83,7 @@ public:
explicit RemoveQuota(const std::string& _role);
protected:
- Try<bool> perform(
- Registry* registry,
- hashset<SlaveID>* slaveIDs,
- bool strict);
+ Try<bool> perform(Registry* registry, hashset<SlaveID>* slaveIDs);
private:
const std::string role;
http://git-wip-us.apache.org/repos/asf/mesos/blob/ef9211f0/src/master/registrar.cpp
----------------------------------------------------------------------
diff --git a/src/master/registrar.cpp b/src/master/registrar.cpp
index 7aa0148..529013f 100644
--- a/src/master/registrar.cpp
+++ b/src/master/registrar.cpp
@@ -130,10 +130,7 @@ private:
explicit Recover(const MasterInfo& _info) : info(_info) {}
protected:
- virtual Try<bool> perform(
- Registry* registry,
- hashset<SlaveID>* slaveIDs,
- bool strict)
+ virtual Try<bool> perform(Registry* registry, hashset<SlaveID>* slaveIDs)
{
registry->mutable_master()->mutable_info()->CopyFrom(info);
return true; // Mutation.
@@ -458,7 +455,7 @@ void RegistrarProcess::update()
foreach (Owned<Operation> operation, operations) {
// No need to process the result of the operation.
- (*operation)(®istry, &slaveIDs, flags.registry_strict);
+ (*operation)(®istry, &slaveIDs);
}
LOG(INFO) << "Applied " << operations.size() << " operations in "
http://git-wip-us.apache.org/repos/asf/mesos/blob/ef9211f0/src/master/registrar.hpp
----------------------------------------------------------------------
diff --git a/src/master/registrar.hpp b/src/master/registrar.hpp
index 238a960..a70132b 100644
--- a/src/master/registrar.hpp
+++ b/src/master/registrar.hpp
@@ -52,18 +52,11 @@ public:
// Aided by accumulator(s):
// slaveIDs - is the set of registered slaves.
//
- // NOTE: the "strict" parameter only applies to operations that
- // affect slaves (i.e. registration). See Flags::registry_strict
- // in master/flags.cpp for more information.
- //
// Returns whether the operation mutates 'registry', or an error if
// the operation cannot be applied successfully.
- Try<bool> operator()(
- Registry* registry,
- hashset<SlaveID>* slaveIDs,
- bool strict)
+ Try<bool> operator()(Registry* registry, hashset<SlaveID>* slaveIDs)
{
- const Try<bool> result = perform(registry, slaveIDs, strict);
+ const Try<bool> result = perform(registry, slaveIDs);
success = !result.isError();
@@ -74,10 +67,7 @@ public:
bool set() { return process::Promise<bool>::set(success); }
protected:
- virtual Try<bool> perform(
- Registry* registry,
- hashset<SlaveID>* slaveIDs,
- bool strict) = 0;
+ virtual Try<bool> perform(Registry* registry, hashset<SlaveID>* slaveIDs) = 0;
private:
bool success;
@@ -101,8 +91,6 @@ private:
class Registrar
{
public:
- // If flags.registry_strict is true, all operations will be
- // permitted.
Registrar(const Flags& flags,
mesos::state::protobuf::State* state,
const Option<std::string>& authenticationRealm = None());
http://git-wip-us.apache.org/repos/asf/mesos/blob/ef9211f0/src/master/weights.cpp
----------------------------------------------------------------------
diff --git a/src/master/weights.cpp b/src/master/weights.cpp
index 4942ac8..d33f2ce 100644
--- a/src/master/weights.cpp
+++ b/src/master/weights.cpp
@@ -25,7 +25,9 @@ UpdateWeights::UpdateWeights(const std::vector<WeightInfo>& _weightInfos)
: weightInfos(_weightInfos) {}
-Try<bool> UpdateWeights::perform(Registry* registry, hashset<SlaveID>*, bool)
+Try<bool> UpdateWeights::perform(
+ Registry* registry,
+ hashset<SlaveID>* /*slaveIDs*/)
{
bool mutated = false;
if (weightInfos.empty()) {
http://git-wip-us.apache.org/repos/asf/mesos/blob/ef9211f0/src/master/weights.hpp
----------------------------------------------------------------------
diff --git a/src/master/weights.hpp b/src/master/weights.hpp
index 80d1369..4b9c438 100644
--- a/src/master/weights.hpp
+++ b/src/master/weights.hpp
@@ -41,9 +41,6 @@ namespace weights {
// will render the operation hanging (i.e. `Future` for the operation
// will not be set).
-// The `strict` flag is not relevant for weights update operations: they will
-// always succeed, even if the flag is set to `true`.
-
/**
* Updates weights for the specified roles. No assumptions are made here:
* the roles may be unknown to the master, or weights can be already set
@@ -56,7 +53,7 @@ public:
explicit UpdateWeights(const std::vector<WeightInfo>& _weightInfos);
protected:
- Try<bool> perform(Registry* registry, hashset<SlaveID>*, bool);
+ Try<bool> perform(Registry* registry, hashset<SlaveID>* slaveIDs);
private:
const std::vector<WeightInfo> weightInfos;
[21/27] mesos git commit: Ignored agent registrations with duplicate
agent IDs.
Posted by vi...@apache.org.
Ignored agent registrations with duplicate agent IDs.
If an agent attempts to register and is assigned an agent ID that
already appears in the registry, the likely cause is a UUID collision:
slave IDs are prefixed with master IDs, and master IDs are randomly
generated UUIDs. UUID collisions are extremely unlikely, so in practice
this should never happen.
This commit updates the comments to clarify how unlikely this situation
is. Furthermore, when it does happen, the master now just ignores the
registration attempt, rather than trying to shutdown the agent. This is
simpler; the agent will eventually try to register again and be assigned
a new agent ID. Finally, it is unclear that shutting down the agent is
actually appropriate: the previous coding added the duplicate ID to
`slaves.removed`, which will interfere with the activity of the other
slave whose ID we collided with.
Review: https://reviews.apache.org/r/51913/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/d9d6f7cc
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/d9d6f7cc
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/d9d6f7cc
Branch: refs/heads/master
Commit: d9d6f7ccf4dbd3826a687fc1c348a8e2f952ed58
Parents: aae8196
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:49:07 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:49:07 2016 -0700
----------------------------------------------------------------------
src/master/master.cpp | 72 +++++++++++++++++++++++-----------------------
src/master/master.hpp | 5 +++-
2 files changed, 40 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/d9d6f7cc/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index e4499b5..20ccd8b 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -5079,50 +5079,50 @@ void Master::_registerSlave(
if (admit.isFailed()) {
LOG(FATAL) << "Failed to admit agent " << slaveInfo.id() << " at " << pid
<< " (" << slaveInfo.hostname() << "): " << admit.failure();
- } else if (!admit.get()) {
- // This means the slave is already present in the registrar, it's
- // likely we generated a duplicate slave id!
+ }
+
+ if (!admit.get()) {
+ // This should only happen if there is a slaveID collision, but that
+ // is extremely unlikely in practice: slaveIDs are prefixed with the
+ // master ID, which is a randomly generated UUID. In this situation,
+ // we ignore the registration attempt. The slave will eventually try
+ // to register again and be assigned a new slave ID.
LOG(ERROR) << "Agent " << slaveInfo.id() << " at " << pid
- << " (" << slaveInfo.hostname() << ") was not admitted, "
- << "asking to shut down";
- slaves.removed.put(slaveInfo.id(), Nothing());
+ << " (" << slaveInfo.hostname() << ") was assigned"
+ << " an agent ID that already appears in the registry;"
+ << " ignoring registration attempt";
+ return;
+ }
- ShutdownMessage message;
- message.set_message(
- "Agent attempted to register but got duplicate agent id " +
- stringify(slaveInfo.id()));
- send(pid, message);
- } else {
- MachineID machineId;
- machineId.set_hostname(slaveInfo.hostname());
- machineId.set_ip(stringify(pid.address.ip));
+ MachineID machineId;
+ machineId.set_hostname(slaveInfo.hostname());
+ machineId.set_ip(stringify(pid.address.ip));
- Slave* slave = new Slave(
- this,
- slaveInfo,
- pid,
- machineId,
- version,
- Clock::now(),
- checkpointedResources);
+ Slave* slave = new Slave(
+ this,
+ slaveInfo,
+ pid,
+ machineId,
+ version,
+ Clock::now(),
+ checkpointedResources);
- ++metrics->slave_registrations;
+ ++metrics->slave_registrations;
- addSlave(slave);
+ addSlave(slave);
- Duration pingTimeout =
- flags.agent_ping_timeout * flags.max_agent_ping_timeouts;
- MasterSlaveConnection connection;
- connection.set_total_ping_timeout_seconds(pingTimeout.secs());
+ Duration pingTimeout =
+ flags.agent_ping_timeout * flags.max_agent_ping_timeouts;
+ MasterSlaveConnection connection;
+ connection.set_total_ping_timeout_seconds(pingTimeout.secs());
- SlaveRegisteredMessage message;
- message.mutable_slave_id()->CopyFrom(slave->id);
- message.mutable_connection()->CopyFrom(connection);
- send(slave->pid, message);
+ SlaveRegisteredMessage message;
+ message.mutable_slave_id()->CopyFrom(slave->id);
+ message.mutable_connection()->CopyFrom(connection);
+ send(slave->pid, message);
- LOG(INFO) << "Registered agent " << *slave
- << " with " << slave->info.resources();
- }
+ LOG(INFO) << "Registered agent " << *slave
+ << " with " << slave->info.resources();
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/d9d6f7cc/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index 4ba8e53..dbcaa3d 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -1934,7 +1934,10 @@ protected:
hashset<SlaveID>* slaveIDs,
bool strict)
{
- // Check and see if this slave is currently admitted.
+ // Check if this slave is currently admitted. This should only
+ // happen if there is a slaveID collision, but that is extremely
+ // unlikely in practice: slaveIDs are prefixed with the master ID,
+ // which is a randomly generated UUID.
if (slaveIDs->contains(info.id())) {
if (strict) {
return Error("Agent already admitted");
[22/27] mesos git commit: Changed registrar operations to no longer
depend on "strict" flag.
Posted by vi...@apache.org.
Changed registrar operations to no longer depend on "strict" flag.
Registrar operations to admit and remove slaves now behave the same way,
regardless of the value of the "strict" flag.
This commit changes the behavior of these operations by enabling the
previous "strict" behavior. For example, removing a slave that is not
registered results in an `Error`, rather than silently ignoring the
problem. However, the master should never attempt to remove a slave that
is not registered, so this is a useful invariant to check. Similarly,
attempting to admit a slave that is already admitted will now result in
an `Error`; this should only occur when there is a slave ID collision,
so again it seems appropriate to produce an error rather than ignoring
the problem.
Review: https://reviews.apache.org/r/51953/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/9fa99618
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/9fa99618
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/9fa99618
Branch: refs/heads/master
Commit: 9fa99618eec85624f44b026804c89be509e18f26
Parents: d9d6f7c
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:49:13 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:49:13 2016 -0700
----------------------------------------------------------------------
src/master/master.cpp | 2 ++
src/master/master.hpp | 22 ++++++++--------------
src/tests/registrar_tests.cpp | 25 ++++---------------------
3 files changed, 14 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/9fa99618/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 20ccd8b..763c5e7 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -7372,6 +7372,8 @@ void Master::_removeSlave(
<< " from the registrar: " << registrarResult.failure();
}
+ // Should not happen: the master will only try to remove agents that
+ // are currently admitted.
CHECK(registrarResult.get())
<< "Agent " << slave->info.id() << " (" << slave->info.hostname() << ") "
<< "already removed from the registrar";
http://git-wip-us.apache.org/repos/asf/mesos/blob/9fa99618/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index dbcaa3d..a051742 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -1932,18 +1932,14 @@ protected:
virtual Try<bool> perform(
Registry* registry,
hashset<SlaveID>* slaveIDs,
- bool strict)
+ bool)
{
// Check if this slave is currently admitted. This should only
// happen if there is a slaveID collision, but that is extremely
// unlikely in practice: slaveIDs are prefixed with the master ID,
// which is a randomly generated UUID.
if (slaveIDs->contains(info.id())) {
- if (strict) {
- return Error("Agent already admitted");
- } else {
- return false; // No mutation.
- }
+ return Error("Agent already admitted");
}
Registry::Slave* slave = registry->mutable_slaves()->add_slaves();
@@ -1971,7 +1967,7 @@ protected:
virtual Try<bool> perform(
Registry* registry,
hashset<SlaveID>* slaveIDs,
- bool strict)
+ bool)
{
// As currently implemented, this should not be possible: the
// master will only mark slaves unreachable that are currently
@@ -2024,7 +2020,7 @@ protected:
virtual Try<bool> perform(
Registry* registry,
hashset<SlaveID>* slaveIDs,
- bool strict)
+ bool)
{
// A slave might try to reregister that appears in the list of
// admitted slaves. This can occur when the master fails over:
@@ -2125,7 +2121,7 @@ protected:
virtual Try<bool> perform(
Registry* registry,
hashset<SlaveID>* slaveIDs,
- bool strict)
+ bool)
{
for (int i = 0; i < registry->slaves().slaves().size(); i++) {
const Registry::Slave& slave = registry->slaves().slaves(i);
@@ -2136,11 +2132,9 @@ protected:
}
}
- if (strict) {
- return Error("Agent not yet admitted");
- } else {
- return false; // No mutation.
- }
+ // Should not happen: the master will only try to remove agents
+ // that are currently admitted.
+ return Error("Agent not yet admitted");
}
private:
http://git-wip-us.apache.org/repos/asf/mesos/blob/9fa99618/src/tests/registrar_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/registrar_tests.cpp b/src/tests/registrar_tests.cpp
index 928bc66..6c42fd1 100644
--- a/src/tests/registrar_tests.cpp
+++ b/src/tests/registrar_tests.cpp
@@ -274,12 +274,7 @@ TEST_P(RegistrarTest, Admit)
AWAIT_READY(registrar.recover(master));
AWAIT_TRUE(registrar.apply(Owned<Operation>(new AdmitSlave(slave))));
-
- if (flags.registry_strict) {
- AWAIT_FALSE(registrar.apply(Owned<Operation>(new AdmitSlave(slave))));
- } else {
- AWAIT_TRUE(registrar.apply(Owned<Operation>(new AdmitSlave(slave))));
- }
+ AWAIT_FALSE(registrar.apply(Owned<Operation>(new AdmitSlave(slave))));
}
@@ -436,29 +431,17 @@ TEST_P(RegistrarTest, Remove)
AWAIT_TRUE(registrar.apply(Owned<Operation>(new RemoveSlave(info1))));
- if (flags.registry_strict) {
- AWAIT_FALSE(registrar.apply(Owned<Operation>(new RemoveSlave(info1))));
- } else {
- AWAIT_TRUE(registrar.apply(Owned<Operation>(new RemoveSlave(info1))));
- }
+ AWAIT_FALSE(registrar.apply(Owned<Operation>(new RemoveSlave(info1))));
AWAIT_TRUE(registrar.apply(Owned<Operation>(new AdmitSlave(info1))));
AWAIT_TRUE(registrar.apply(Owned<Operation>(new RemoveSlave(info2))));
- if (flags.registry_strict) {
- AWAIT_FALSE(registrar.apply(Owned<Operation>(new RemoveSlave(info2))));
- } else {
- AWAIT_TRUE(registrar.apply(Owned<Operation>(new RemoveSlave(info2))));
- }
+ AWAIT_FALSE(registrar.apply(Owned<Operation>(new RemoveSlave(info2))));
AWAIT_TRUE(registrar.apply(Owned<Operation>(new RemoveSlave(info3))));
- if (flags.registry_strict) {
- AWAIT_FALSE(registrar.apply(Owned<Operation>(new RemoveSlave(info3))));
- } else {
- AWAIT_TRUE(registrar.apply(Owned<Operation>(new RemoveSlave(info3))));
- }
+ AWAIT_FALSE(registrar.apply(Owned<Operation>(new RemoveSlave(info3))));
}
[15/27] mesos git commit: Handled agents failing health checks
multiple times.
Posted by vi...@apache.org.
Handled agents failing health checks multiple times.
Now that we wait for the agent to be removed from the registry before
stopping the SlaveObserver, it is possible for an agent to fail health
checks multiple times if the registry operation takes longer than
`agent_ping_timeout`.
This commit updates the master logic to handle this by ignoring health
check failures while the registry operation to mark the agent
unreachable is still in progress.
Review: https://reviews.apache.org/r/51653/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/5e66002c
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/5e66002c
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/5e66002c
Branch: refs/heads/master
Commit: 5e66002cec7855055245e328c34a783a55c542f6
Parents: 0d85e7e
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:48:30 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:48:30 2016 -0700
----------------------------------------------------------------------
src/master/master.cpp | 12 ++++
src/tests/partition_tests.cpp | 124 +++++++++++++++++++++++++++++++++++++
2 files changed, 136 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/5e66002c/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 4e0df8a..3095f8a 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -5827,6 +5827,18 @@ void Master::markUnreachable(const SlaveID& slaveId)
return;
}
+ if (slaves.markingUnreachable.contains(slaveId)) {
+ // Possible if marking the slave unreachable in the registry takes
+ // a long time. While the registry operation is in progress, the
+ // `SlaveObserver` will continue to ping the slave; if the slave
+ // fails another health check, the `SlaveObserver` will trigger
+ // another attempt to mark it unreachable.
+ LOG(WARNING) << "Not marking agent " << slaveId
+ << " unreachable because another unreachable"
+ << " transition is already in progress";
+ return;
+ }
+
LOG(INFO) << "Marking agent " << *slave
<< " unreachable: health check timed out";
http://git-wip-us.apache.org/repos/asf/mesos/blob/5e66002c/src/tests/partition_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/partition_tests.cpp b/src/tests/partition_tests.cpp
index 54d07b7..d2fce57 100644
--- a/src/tests/partition_tests.cpp
+++ b/src/tests/partition_tests.cpp
@@ -58,6 +58,7 @@ using process::Future;
using process::Message;
using process::Owned;
using process::PID;
+using process::Promise;
using process::Time;
using process::http::OK;
@@ -67,6 +68,7 @@ using std::vector;
using testing::_;
using testing::AtMost;
+using testing::DoAll;
using testing::Eq;
using testing::Return;
@@ -1869,6 +1871,128 @@ TEST_P(PartitionTest, RegistryGcByAge)
}
+// This test checks that the master behaves correctly if a slave fails
+// health checks twice. At present, this can only occur if the registry
+// operation to mark the slave unreachable takes so long that the
+// slave fails an additional health check in the mean time.
+TEST_P(PartitionTest, FailHealthChecksTwice)
+{
+ master::Flags masterFlags = CreateMasterFlags();
+ masterFlags.registry_strict = GetParam();
+
+ Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
+ ASSERT_SOME(master);
+
+ // Set these expectations up before we spawn the slave so that we
+ // don't miss the first PING.
+ Future<Message> ping = FUTURE_MESSAGE(
+ Eq(PingSlaveMessage().GetTypeName()), _, _);
+
+ // Drop all the PONGs to simulate slave partition.
+ DROP_PROTOBUFS(PongSlaveMessage(), _, _);
+
+ Owned<MasterDetector> detector = master.get()->createDetector();
+ Try<Owned<cluster::Slave>> slave = StartSlave(detector.get());
+ ASSERT_SOME(slave);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(&driver, _, _));
+
+ Future<Nothing> resourceOffers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureSatisfy(&resourceOffers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ // Need to make sure the framework AND slave have registered with
+ // master. Waiting for resource offers should accomplish both.
+ AWAIT_READY(resourceOffers);
+
+ Clock::pause();
+
+ EXPECT_CALL(sched, offerRescinded(&driver, _))
+ .Times(AtMost(1));
+
+ Future<Nothing> slaveLost;
+ EXPECT_CALL(sched, slaveLost(&driver, _))
+ .WillOnce(FutureSatisfy(&slaveLost));
+
+ // Now advance through the PINGs.
+ size_t pings = 0;
+ while (true) {
+ AWAIT_READY(ping);
+ pings++;
+ if (pings == masterFlags.max_agent_ping_timeouts) {
+ break;
+ }
+ ping = FUTURE_MESSAGE(Eq(PingSlaveMessage().GetTypeName()), _, _);
+ Clock::advance(masterFlags.agent_ping_timeout);
+ }
+
+ // The slave observer should dispatch a message to the master. We
+ // intercept the next registrar operation; this should be the
+ // registry operation marking the slave unreachable.
+ Future<Nothing> unreachableDispatch1 =
+ FUTURE_DISPATCH(master.get()->pid, &Master::markUnreachable);
+
+ Future<Owned<master::Operation>> markUnreachable;
+ Promise<bool> markUnreachableContinue;
+ EXPECT_CALL(*master.get()->registrar.get(), apply(_))
+ .WillOnce(DoAll(FutureArg<0>(&markUnreachable),
+ Return(markUnreachableContinue.future())));
+
+ Clock::advance(masterFlags.agent_ping_timeout);
+
+ AWAIT_READY(unreachableDispatch1);
+ AWAIT_READY(markUnreachable);
+ EXPECT_NE(
+ nullptr,
+ dynamic_cast<master::MarkSlaveUnreachable*>(
+ markUnreachable.get().get()));
+
+ // Cause the slave to fail another health check. This is possible
+ // because we don't shutdown the SlaveObserver until we have marked
+ // the slave as unreachable in the registry. The second health check
+ // failure should dispatch to the master but this should NOT result
+ // in another registry operation.
+ Future<Nothing> unreachableDispatch2 =
+ FUTURE_DISPATCH(master.get()->pid, &Master::markUnreachable);
+
+ EXPECT_CALL(*master.get()->registrar.get(), apply(_))
+ .Times(0);
+
+ Clock::advance(masterFlags.agent_ping_timeout);
+
+ AWAIT_READY(unreachableDispatch2);
+
+ // Apply the registry operation, then pass the result back to the
+ // master to allow it to continue.
+ Future<bool> applyUnreachable =
+ master.get()->registrar->unmocked_apply(markUnreachable.get());
+
+ AWAIT_READY(applyUnreachable);
+ markUnreachableContinue.set(applyUnreachable.get());
+
+ AWAIT_READY(slaveLost);
+
+ JSON::Object stats = Metrics();
+ EXPECT_EQ(2, stats.values["master/slave_unreachable_scheduled"]);
+ EXPECT_EQ(2, stats.values["master/slave_unreachable_completed"]);
+ EXPECT_EQ(1, stats.values["master/slave_removals"]);
+ EXPECT_EQ(1, stats.values["master/slave_removals/reason_unhealthy"]);
+ EXPECT_EQ(0, stats.values["master/slave_removals/reason_unregistered"]);
+
+ driver.stop();
+ driver.join();
+
+ Clock::resume();
+}
+
+
class OneWayPartitionTest : public MesosTest {};
[09/27] mesos git commit: Change registry update order on removal,
mark-unreachable.
Posted by vi...@apache.org.
Change registry update order on removal, mark-unreachable.
This commit changes the master so that we always update the registry
for an important change (e.g., removing an agent or marking an agent
unreachable), before updating the important parts of the master's
in-memory state (e.g., the in-memory list of registered agents).
Previously, the registry was updated first for registration and
reregistration, but second for removal and marking unreachable.
Updating the registry first is simpler, and also avoids possibly
leaking inaccurate information (e.g., via HTTP endpoints) if the
registry operation fails.
Review: https://reviews.apache.org/r/51374/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/88b2533a
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/88b2533a
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/88b2533a
Branch: refs/heads/master
Commit: 88b2533a72ff5897bc6cc5d7861a313c6fd7ea28
Parents: 0cb2480
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:47:53 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:47:53 2016 -0700
----------------------------------------------------------------------
src/master/master.cpp | 332 ++++++++++++++++++-------------------
src/master/master.hpp | 16 +-
src/tests/master_tests.cpp | 1 +
src/tests/partition_tests.cpp | 26 +--
src/tests/slave_tests.cpp | 1 +
5 files changed, 193 insertions(+), 183 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/88b2533a/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 008ff32..71caea6 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -1757,21 +1757,20 @@ Nothing Master::markUnreachableAfterFailover(const Registry::Slave& slave)
<< " after master failover; marking it unreachable";
++metrics->slave_unreachable_completed;
- ++metrics->recovery_slave_removals;
slaves.recovered.erase(slave.info().id());
+ TimeInfo unreachableTime = protobuf::getCurrentTime();
+
if (flags.registry_strict) {
slaves.markingUnreachable.insert(slave.info().id());
registrar->apply(Owned<Operation>(
- new MarkSlaveUnreachable(slave.info(), protobuf::getCurrentTime())))
+ new MarkSlaveUnreachable(slave.info(), unreachableTime)))
.onAny(defer(self(),
- &Self::_markUnreachable,
+ &Self::_markUnreachableAfterFailover,
slave.info(),
- vector<StatusUpdate>(), // No TASK_LOST updates to send.
- lambda::_1,
- "did not re-register after master failover"));
+ lambda::_1));
} else {
// When a non-strict registry is in use, we want to ensure the
// registry is used in a write-only manner. Therefore we remove
@@ -1781,7 +1780,7 @@ Nothing Master::markUnreachableAfterFailover(const Registry::Slave& slave)
"Failed to mark agent " + stringify(slave.info().id()) + " unreachable";
registrar->apply(Owned<Operation>(
- new MarkSlaveUnreachable(slave.info(), protobuf::getCurrentTime())))
+ new MarkSlaveUnreachable(slave.info(), unreachableTime)))
.onFailed(lambda::bind(fail, message, lambda::_1));
}
@@ -1789,6 +1788,54 @@ Nothing Master::markUnreachableAfterFailover(const Registry::Slave& slave)
}
+void Master::_markUnreachableAfterFailover(
+ const SlaveInfo& slaveInfo,
+ const Future<bool>& registrarResult)
+{
+ CHECK(slaves.markingUnreachable.contains(slaveInfo.id()));
+ slaves.markingUnreachable.erase(slaveInfo.id());
+
+ if (registrarResult.isFailed()) {
+ LOG(FATAL) << "Failed to mark agent " << slaveInfo.id()
+ << " (" << slaveInfo.hostname() << ")"
+ << " unreachable in the registry: "
+ << registrarResult.failure();
+ }
+
+ CHECK(!registrarResult.isDiscarded());
+
+ // `MarkSlaveUnreachable` registry operation should never fail.
+ CHECK(registrarResult.get());
+
+ LOG(INFO) << "Marked agent " << slaveInfo.id() << " ("
+ << slaveInfo.hostname() << ") unreachable: "
+ << "did not re-register after master failover";
+
+ ++metrics->slave_removals;
+ ++metrics->slave_removals_reason_unhealthy;
+ ++metrics->recovery_slave_removals;
+
+ sendSlaveLost(slaveInfo);
+}
+
+
+void Master::sendSlaveLost(const SlaveInfo& slaveInfo)
+{
+ foreachvalue (Framework* framework, frameworks.registered) {
+ LOG(INFO) << "Notifying framework " << *framework << " of lost agent "
+ << slaveInfo.id() << " (" << slaveInfo.hostname() << ")";
+
+ LostSlaveMessage message;
+ message.mutable_slave_id()->MergeFrom(slaveInfo.id());
+ framework->send(message);
+ }
+
+ if (HookManager::hooksAvailable()) {
+ HookManager::masterSlaveLostHook(slaveInfo);
+ }
+}
+
+
void Master::fileAttached(const Future<Nothing>& result, const string& path)
{
if (result.isReady()) {
@@ -5679,6 +5726,54 @@ void Master::markUnreachable(const SlaveID& slaveId)
CHECK(!slaves.unreachable.contains(slaveId));
CHECK(slaves.removed.get(slaveId).isNone());
+ slaves.markingUnreachable.insert(slave->id);
+
+ // Use the same timestamp for all status updates sent below; we also
+ // use this timestamp when updating the registry.
+ TimeInfo unreachableTime = protobuf::getCurrentTime();
+
+ // Update the registry to move this slave from the list of admitted
+ // slaves to the list of unreachable slaves. After this is
+ // completed, we can update the master's in-memory state to remove
+ // the slave and send TASK_LOST status updates to the frameworks.
+ registrar->apply(Owned<Operation>(
+ new MarkSlaveUnreachable(slave->info, unreachableTime)))
+ .onAny(defer(self(),
+ &Self::_markUnreachable,
+ slave,
+ unreachableTime,
+ lambda::_1));
+}
+
+
+void Master::_markUnreachable(
+ Slave* slave,
+ TimeInfo unreachableTime,
+ const Future<bool>& registrarResult)
+{
+ CHECK_NOTNULL(slave);
+ CHECK(slaves.markingUnreachable.contains(slave->info.id()));
+ slaves.markingUnreachable.erase(slave->info.id());
+
+ if (registrarResult.isFailed()) {
+ LOG(FATAL) << "Failed to mark agent " << slave->info.id()
+ << " (" << slave->info.hostname() << ")"
+ << " unreachable in the registry: "
+ << registrarResult.failure();
+ }
+
+ CHECK(!registrarResult.isDiscarded());
+
+ // `MarkSlaveUnreachable` registry operation should never fail.
+ CHECK(registrarResult.get());
+
+ LOG(INFO) << "Marked agent " << slave->info.id() << " ("
+ << slave->info.hostname() << ") unreachable: "
+ << "health check timed out";
+
+ ++metrics->slave_removals;
+ ++metrics->slave_removals_reason_unhealthy;
+
// We want to remove the slave first, to avoid the allocator
// re-allocating the recovered resources.
//
@@ -5689,17 +5784,9 @@ void Master::markUnreachable(const SlaveID& slaveId)
// the slave is already removed.
allocator->removeSlave(slave->id);
- // Use the same timestamp for all status updates sent below; we also
- // use this timestamp when updating the registry.
- TimeInfo unreachableTime = protobuf::getCurrentTime();
-
- // Transition the tasks to TASK_LOST and remove them, BUT do not
- // send updates yet. Rather, build up the updates so that we can can
- // send them after the slave has been moved to the unreachable list
- // in the registry.
+ // Transition the tasks to TASK_LOST and remove them.
// TODO(neilc): Update this to send TASK_UNREACHABLE for
- // partition-aware frameworks.
- vector<StatusUpdate> updates;
+ // PARTITION_AWARE frameworks.
foreachkey (const FrameworkID& frameworkId, utils::copy(slave->tasks)) {
Framework* framework = getFramework(frameworkId);
CHECK_NOTNULL(framework);
@@ -5724,7 +5811,7 @@ void Master::markUnreachable(const SlaveID& slaveId)
updateTask(task, update);
removeTask(task);
- updates.push_back(update);
+ forward(update, UPID(), framework);
}
}
@@ -5759,7 +5846,6 @@ void Master::markUnreachable(const SlaveID& slaveId)
slaves.registered.remove(slave);
slaves.removed.put(slave->id, Nothing());
slaves.unreachable[slave->id] = unreachableTime;
- slaves.markingUnreachable.insert(slave->id);
authenticated.erase(slave->pid);
// Remove the slave from the `machines` mapping.
@@ -5774,81 +5860,12 @@ void Master::markUnreachable(const SlaveID& slaveId)
// TODO(benh): unlink(slave->pid);
- // Update the registry to move this slave from the list of admitted
- // slaves to the list of unreachable slaves. Once this is completed,
- // we can forward the TASK_LOST updates to the frameworks.
- registrar->apply(Owned<Operation>(
- new MarkSlaveUnreachable(slave->info, unreachableTime)))
- .onAny(defer(self(),
- &Self::_markUnreachable,
- slave->info,
- updates,
- lambda::_1,
- "health check timed out"));
+ sendSlaveLost(slave->info);
delete slave;
}
-void Master::_markUnreachable(
- const SlaveInfo& slaveInfo,
- const vector<StatusUpdate>& updates,
- const Future<bool>& registrarResult,
- const string& unreachableCause)
-{
- CHECK(slaves.markingUnreachable.contains(slaveInfo.id()));
- slaves.markingUnreachable.erase(slaveInfo.id());
-
- if (registrarResult.isFailed()) {
- LOG(FATAL) << "Failed to mark agent " << slaveInfo.id()
- << " (" << slaveInfo.hostname() << ")"
- << " unreachable in the registry: "
- << registrarResult.failure();
- }
-
- CHECK(!registrarResult.isDiscarded());
-
- // `MarkSlaveUnreachable` registry operation should never fail.
- CHECK(registrarResult.get());
-
- LOG(INFO) << "Marked agent " << slaveInfo.id() << " ("
- << slaveInfo.hostname() << ") unreachable: "
- << unreachableCause;
-
- // TODO(neilc): Consider renaming these metrics or adding new
- // metrics for the new PARTITION_AWARE semantics.
- ++metrics->slave_removals;
- ++metrics->slave_removals_reason_unhealthy;
-
- // Forward the TASK_LOST updates on to the frameworks.
- foreach (const StatusUpdate& update, updates) {
- Framework* framework = getFramework(update.framework_id());
-
- if (framework == nullptr) {
- LOG(WARNING) << "Dropping update " << update << " from unknown framework "
- << update.framework_id();
- } else {
- forward(update, UPID(), framework);
- }
- }
-
- // Notify all frameworks of the lost slave.
- foreachvalue (Framework* framework, frameworks.registered) {
- LOG(INFO) << "Notifying framework " << *framework << " of lost agent "
- << slaveInfo.id() << " (" << slaveInfo.hostname() << ")";
-
- LostSlaveMessage message;
- message.mutable_slave_id()->MergeFrom(slaveInfo.id());
- framework->send(message);
- }
-
- // Finally, notify the `SlaveLost` hooks.
- if (HookManager::hooksAvailable()) {
- HookManager::masterSlaveLostHook(slaveInfo);
- }
-}
-
-
void Master::reconcile(
Framework* framework,
const scheduler::Call::Reconcile& reconcile)
@@ -7150,6 +7167,55 @@ void Master::removeSlave(
LOG(INFO) << "Removing agent " << *slave << ": " << message;
+ slaves.removing.insert(slave->id);
+
+ // Remove this slave from the registrar. Note that we update the
+ // registry BEFORE we update the master's in-memory state; this
+ // means that until the registry operation has completed, the slave
+ // is not considered to be removed (so we might offer its resources
+ // to frameworks, etc.). Ensuring that the registry update succeeds
+ // before we modify in-memory state ensures that external clients
+ // see consistent behavior if the master fails over.
+ registrar->apply(Owned<Operation>(new RemoveSlave(slave->info)))
+ .onAny(defer(self(),
+ &Self::_removeSlave,
+ slave,
+ lambda::_1,
+ message,
+ reason));
+}
+
+
+void Master::_removeSlave(
+ Slave* slave,
+ const Future<bool>& registrarResult,
+ const string& removalCause,
+ Option<Counter> reason)
+{
+ CHECK_NOTNULL(slave);
+ CHECK(slaves.removing.contains(slave->info.id()));
+ slaves.removing.erase(slave->info.id());
+
+ CHECK(!registrarResult.isDiscarded());
+
+ if (registrarResult.isFailed()) {
+ LOG(FATAL) << "Failed to remove agent " << slave->info.id()
+ << " (" << slave->info.hostname() << ")"
+ << " from the registrar: " << registrarResult.failure();
+ }
+
+ CHECK(registrarResult.get())
+ << "Agent " << slave->info.id() << " (" << slave->info.hostname() << ") "
+ << "already removed from the registrar";
+
+ LOG(INFO) << "Removed agent " << slave->info.id() << " ("
+ << slave->info.hostname() << "): " << removalCause;
+
+ ++metrics->slave_removals;
+ if (reason.isSome()) {
+ ++utils::copy(reason.get()); // Remove const.
+ }
+
// We want to remove the slave first, to avoid the allocator
// re-allocating the recovered resources.
//
@@ -7160,10 +7226,7 @@ void Master::removeSlave(
// the slave is already removed.
allocator->removeSlave(slave->id);
- // Transition the tasks to lost and remove them, BUT do not send
- // updates. Rather, build up the updates so that we can send them
- // after the slave is removed from the registry.
- vector<StatusUpdate> updates;
+ // Transition the tasks to lost and remove them.
foreachkey (const FrameworkID& frameworkId, utils::copy(slave->tasks)) {
foreachvalue (Task* task, utils::copy(slave->tasks[frameworkId])) {
const StatusUpdate& update = protobuf::createStatusUpdate(
@@ -7173,7 +7236,7 @@ void Master::removeSlave(
TASK_LOST,
TaskStatus::SOURCE_MASTER,
None(),
- "Slave " + slave->info.hostname() + " removed: " + message,
+ "Slave " + slave->info.hostname() + " removed: " + removalCause,
TaskStatus::REASON_SLAVE_REMOVED,
(task->has_executor_id() ?
Option<ExecutorID>(task->executor_id()) : None()));
@@ -7181,7 +7244,13 @@ void Master::removeSlave(
updateTask(task, update);
removeTask(task);
- updates.push_back(update);
+ Framework* framework = getFramework(frameworkId);
+ if (framework == nullptr) {
+ LOG(WARNING) << "Dropping update " << update
+ << " for unknown framework " << frameworkId;
+ } else {
+ forward(update, UPID(), framework);
+ }
}
}
@@ -7216,7 +7285,6 @@ void Master::removeSlave(
slave->pendingTasks.clear();
// Mark the slave as being removed.
- slaves.removing.insert(slave->id);
slaves.registered.remove(slave);
slaves.removed.put(slave->id, Nothing());
authenticated.erase(slave->pid);
@@ -7233,82 +7301,12 @@ void Master::removeSlave(
// TODO(benh): unlink(slave->pid);
- // Remove this slave from the registrar. Once this is completed, we
- // can forward the LOST task updates to the frameworks and notify
- // all frameworks that this slave was lost.
- registrar->apply(Owned<Operation>(new RemoveSlave(slave->info)))
- .onAny(defer(self(),
- &Self::_removeSlave,
- slave->info,
- updates,
- lambda::_1,
- message,
- reason));
+ sendSlaveLost(slave->info);
delete slave;
}
-void Master::_removeSlave(
- const SlaveInfo& slaveInfo,
- const vector<StatusUpdate>& updates,
- const Future<bool>& registrarResult,
- const string& removalCause,
- Option<Counter> reason)
-{
- CHECK(slaves.removing.contains(slaveInfo.id()));
- slaves.removing.erase(slaveInfo.id());
-
- CHECK(!registrarResult.isDiscarded());
-
- if (registrarResult.isFailed()) {
- LOG(FATAL) << "Failed to remove agent " << slaveInfo.id()
- << " (" << slaveInfo.hostname() << ")"
- << " from the registrar: " << registrarResult.failure();
- }
-
- CHECK(registrarResult.get())
- << "Agent " << slaveInfo.id() << " (" << slaveInfo.hostname() << ") "
- << "already removed from the registrar";
-
- LOG(INFO) << "Removed agent " << slaveInfo.id() << " ("
- << slaveInfo.hostname() << "): " << removalCause;
-
- ++metrics->slave_removals;
-
- if (reason.isSome()) {
- ++utils::copy(reason.get()); // Remove const.
- }
-
- // Forward the LOST updates on to the framework.
- foreach (const StatusUpdate& update, updates) {
- Framework* framework = getFramework(update.framework_id());
-
- if (framework == nullptr) {
- LOG(WARNING) << "Dropping update " << update << " from unknown framework "
- << update.framework_id();
- } else {
- forward(update, UPID(), framework);
- }
- }
-
- // Notify all frameworks of the lost slave.
- foreachvalue (Framework* framework, frameworks.registered) {
- LOG(INFO) << "Notifying framework " << *framework << " of lost agent "
- << slaveInfo.id() << " (" << slaveInfo.hostname() << ")";
-
- LostSlaveMessage message;
- message.mutable_slave_id()->MergeFrom(slaveInfo.id());
- framework->send(message);
- }
-
- // Finally, notify the `SlaveLost` hooks.
- if (HookManager::hooksAvailable()) {
- HookManager::masterSlaveLostHook(slaveInfo);
- }
-}
-
-
void Master::updateTask(Task* task, const StatusUpdate& update)
{
CHECK_NOTNULL(task);
http://git-wip-us.apache.org/repos/asf/mesos/blob/88b2533a/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index 7e50359..714aa79 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -685,15 +685,20 @@ protected:
std::vector<Archive::Framework>());
void _markUnreachable(
- const SlaveInfo& slaveInfo,
- const std::vector<StatusUpdate>& updates,
- const process::Future<bool>& registrarResult,
- const std::string& unreachableCause);
+ Slave* slave,
+ TimeInfo unreachableTime,
+ const process::Future<bool>& registrarResult);
// Mark a slave as unreachable in the registry. Called when the slave
// does not re-register in time after a master failover.
Nothing markUnreachableAfterFailover(const Registry::Slave& slave);
+ void _markUnreachableAfterFailover(
+ const SlaveInfo& slaveInfo,
+ const process::Future<bool>& registrarResult);
+
+ void sendSlaveLost(const SlaveInfo& slaveInfo);
+
// Remove the slave from the registrar and from the master's state.
//
// TODO(bmahler): 'reason' is optional until MESOS-2317 is resolved.
@@ -703,8 +708,7 @@ protected:
Option<process::metrics::Counter> reason = None());
void _removeSlave(
- const SlaveInfo& slaveInfo,
- const std::vector<StatusUpdate>& updates,
+ Slave* slave,
const process::Future<bool>& registrarResult,
const std::string& removalCause,
Option<process::metrics::Counter> reason = None());
http://git-wip-us.apache.org/repos/asf/mesos/blob/88b2533a/src/tests/master_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/master_tests.cpp b/src/tests/master_tests.cpp
index 514c1d9..85ebd57 100644
--- a/src/tests/master_tests.cpp
+++ b/src/tests/master_tests.cpp
@@ -1821,6 +1821,7 @@ TEST_F(MasterTest, RecoveredSlaveCanReregister)
EXPECT_EQ(1, stats.values["master/recovery_slave_removals"]);
EXPECT_EQ(1, stats.values["master/slave_removals"]);
EXPECT_EQ(1, stats.values["master/slave_removals/reason_unhealthy"]);
+ EXPECT_EQ(0, stats.values["master/slave_removals/reason_unregistered"]);
EXPECT_EQ(1, stats.values["master/slave_unreachable_completed"]);
EXPECT_EQ(1, stats.values["master/slave_unreachable_scheduled"]);
http://git-wip-us.apache.org/repos/asf/mesos/blob/88b2533a/src/tests/partition_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/partition_tests.cpp b/src/tests/partition_tests.cpp
index f962f84..997eda3 100644
--- a/src/tests/partition_tests.cpp
+++ b/src/tests/partition_tests.cpp
@@ -156,6 +156,7 @@ TEST_P(PartitionTest, PartitionedSlave)
EXPECT_EQ(1, stats.values["master/slave_unreachable_completed"]);
EXPECT_EQ(1, stats.values["master/slave_removals"]);
EXPECT_EQ(1, stats.values["master/slave_removals/reason_unhealthy"]);
+ EXPECT_EQ(0, stats.values["master/slave_removals/reason_unregistered"]);
driver.stop();
driver.join();
@@ -430,6 +431,7 @@ TEST_P(PartitionTest, ReregisterSlaveNotPartitionAware)
EXPECT_EQ(1, stats.values["master/slave_unreachable_completed"]);
EXPECT_EQ(1, stats.values["master/slave_removals"]);
EXPECT_EQ(1, stats.values["master/slave_removals/reason_unhealthy"]);
+ EXPECT_EQ(0, stats.values["master/slave_removals/reason_unregistered"]);
// We now complete the partition on the slave side as well. We
// simulate a master loss event, which would normally happen during
@@ -637,9 +639,10 @@ TEST_P(PartitionTest, PartitionedSlaveReregistrationMasterFailover)
EXPECT_EQ(slaveId, lostStatus.get().slave_id());
EXPECT_EQ(partitionTime, lostStatus.get().unreachable_time());
- // `sched2` should see TASK_UNREACHABLE.
+ // `sched2` should see TASK_LOST.
+ // TODO(neilc): Update this to expect TASK_UNREACHABLE.
AWAIT_READY(unreachableStatus);
- EXPECT_EQ(TASK_UNREACHABLE, unreachableStatus.get().state());
+ EXPECT_EQ(TASK_LOST, unreachableStatus.get().state());
EXPECT_EQ(TaskStatus::REASON_SLAVE_REMOVED, unreachableStatus.get().reason());
EXPECT_EQ(task2.task_id(), unreachableStatus.get().task_id());
EXPECT_EQ(slaveId, unreachableStatus.get().slave_id());
@@ -799,9 +802,9 @@ TEST_P(PartitionTest, PartitionedSlaveOrphanedTask)
// Now, induce a partition of the slave by having the master
// timeout the slave.
- Future<TaskStatus> unreachableStatus;
+ Future<TaskStatus> lostStatus;
EXPECT_CALL(sched, statusUpdate(&driver, _))
- .WillOnce(FutureArg<1>(&unreachableStatus));
+ .WillOnce(FutureArg<1>(&lostStatus));
Future<Nothing> slaveLost;
EXPECT_CALL(sched, slaveLost(&driver, _))
@@ -829,12 +832,13 @@ TEST_P(PartitionTest, PartitionedSlaveOrphanedTask)
Clock::advance(Milliseconds(100));
- AWAIT_READY(unreachableStatus);
- EXPECT_EQ(TASK_UNREACHABLE, unreachableStatus.get().state());
- EXPECT_EQ(TaskStatus::REASON_SLAVE_REMOVED, unreachableStatus.get().reason());
- EXPECT_EQ(task.task_id(), unreachableStatus.get().task_id());
- EXPECT_EQ(slaveId, unreachableStatus.get().slave_id());
- EXPECT_EQ(partitionTime, unreachableStatus.get().unreachable_time());
+ // TODO(neilc): Update this to expect `TASK_UNREACHABLE`.
+ AWAIT_READY(lostStatus);
+ EXPECT_EQ(TASK_LOST, lostStatus.get().state());
+ EXPECT_EQ(TaskStatus::REASON_SLAVE_REMOVED, lostStatus.get().reason());
+ EXPECT_EQ(task.task_id(), lostStatus.get().task_id());
+ EXPECT_EQ(slaveId, lostStatus.get().slave_id());
+ EXPECT_EQ(partitionTime, lostStatus.get().unreachable_time());
AWAIT_READY(slaveLost);
@@ -1079,6 +1083,7 @@ TEST_P(PartitionTest, PartitionedSlaveStatusUpdates)
EXPECT_EQ(1, stats.values["master/slave_unreachable_completed"]);
EXPECT_EQ(1, stats.values["master/slave_removals"]);
EXPECT_EQ(1, stats.values["master/slave_removals/reason_unhealthy"]);
+ EXPECT_EQ(0, stats.values["master/slave_removals/reason_unregistered"]);
// At this point, the slave still thinks it's registered, so we
// simulate a status update coming from the slave.
@@ -1252,6 +1257,7 @@ TEST_P(PartitionTest, PartitionedSlaveExitedExecutor)
EXPECT_EQ(1, stats.values["master/slave_unreachable_completed"]);
EXPECT_EQ(1, stats.values["master/slave_removals"]);
EXPECT_EQ(1, stats.values["master/slave_removals/reason_unhealthy"]);
+ EXPECT_EQ(0, stats.values["master/slave_removals/reason_unregistered"]);
EXPECT_CALL(sched, executorLost(&driver, _, _, _))
.Times(0);
http://git-wip-us.apache.org/repos/asf/mesos/blob/88b2533a/src/tests/slave_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/slave_tests.cpp b/src/tests/slave_tests.cpp
index 2e7accc..aa30118 100644
--- a/src/tests/slave_tests.cpp
+++ b/src/tests/slave_tests.cpp
@@ -165,6 +165,7 @@ TEST_F(SlaveTest, Shutdown)
JSON::Object stats = Metrics();
EXPECT_EQ(1, stats.values["master/slave_removals"]);
EXPECT_EQ(1, stats.values["master/slave_removals/reason_unregistered"]);
+ EXPECT_EQ(0, stats.values["master/slave_removals/reason_unhealthy"]);
driver.stop();
driver.join();
[13/27] mesos git commit: Added GC of unreachable agent metadata from
the registry.
Posted by vi...@apache.org.
Added GC of unreachable agent metadata from the registry.
Information about agents that have been marked unreachable is stored
in the registry. Since the number of unreachable agents can grow without
bound, this commit implements a garbage collection scheme. The current
leading master will periodically examine the registry and discard
information about unreachable agents according to two criteria:
`registry_max_agent_age` and `registry_max_agent_count`. The frequency
with which the master examines the registry is controlled by a third
parameter, `registry_gc_interval`.
Review: https://reviews.apache.org/r/51021/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/9dad0b03
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/9dad0b03
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/9dad0b03
Branch: refs/heads/master
Commit: 9dad0b034c99403cab43cab0a6ea8a55e99e71fa
Parents: 592da1e
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:48:16 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:48:16 2016 -0700
----------------------------------------------------------------------
src/master/constants.hpp | 6 +
src/master/flags.cpp | 31 ++
src/master/flags.hpp | 3 +
src/master/master.cpp | 104 +++++++
src/master/master.hpp | 66 ++++-
src/master/registry.proto | 6 +
src/tests/partition_tests.cpp | 590 +++++++++++++++++++++++++++++++++++++
src/tests/registrar_tests.cpp | 46 +++
8 files changed, 848 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/9dad0b03/src/master/constants.hpp
----------------------------------------------------------------------
diff --git a/src/master/constants.hpp b/src/master/constants.hpp
index cd80dac..5dd0667 100644
--- a/src/master/constants.hpp
+++ b/src/master/constants.hpp
@@ -91,6 +91,12 @@ constexpr Duration WHITELIST_WATCH_INTERVAL = Seconds(5);
// Default number of tasks (limit) for /master/tasks endpoint.
constexpr size_t TASK_LIMIT = 100;
+constexpr Duration DEFAULT_REGISTRY_GC_INTERVAL = Minutes(15);
+
+constexpr Duration DEFAULT_REGISTRY_MAX_AGENT_AGE = Weeks(2);
+
+constexpr size_t DEFAULT_REGISTRY_MAX_AGENT_COUNT = 100 * 1024;
+
/**
* Label used by the Leader Contender and Detector.
*
http://git-wip-us.apache.org/repos/asf/mesos/blob/9dad0b03/src/master/flags.cpp
----------------------------------------------------------------------
diff --git a/src/master/flags.cpp b/src/master/flags.cpp
index 19ae6c1..aea8bd4 100644
--- a/src/master/flags.cpp
+++ b/src/master/flags.cpp
@@ -557,4 +557,35 @@ mesos::internal::master::Flags::Flags()
"should exist in a module specified through the --modules flag.\n"
"Cannot be used in conjunction with --zk.\n"
"Must be used in conjunction with --master_contender.");
+
+ add(&Flags::registry_gc_interval,
+ "registry_gc_interval",
+ "How often to garbage collect the registry. The current leading\n"
+ "master will periodically discard information from the registry.\n"
+ "How long registry state is retained is controlled by other\n"
+ "parameters (e.g., registry_max_agent_age, registry_max_agent_count);\n"
+ "this parameter controls how often the master will examine the\n"
+ "registry to see if data should be discarded.",
+ DEFAULT_REGISTRY_GC_INTERVAL);
+
+ add(&Flags::registry_max_agent_age,
+ "registry_max_agent_age",
+ "Maximum length of time to store information in the registry about\n"
+ "agents that are not currently connected to the cluster. This\n"
+ "information allows frameworks to determine the status of unreachable\n"
+ "and removed agents. Note that the registry always stores information\n"
+ "on all connected agents. If there are more than\n"
+ "`registry_max_agent_count` partitioned or removed agents, agent\n"
+ "information may be discarded from the registry sooner than indicated\n"
+ "by this parameter.",
+ DEFAULT_REGISTRY_MAX_AGENT_AGE);
+
+ add(&Flags::registry_max_agent_count,
+ "registry_max_agent_count",
+ "Maximum number of disconnected agents to store in the registry.\n"
+ "This informtion allows frameworks to determine the status of\n"
+ "disconnected agents. Note that the registry always stores\n"
+ "information about all connected agents. See also the\n"
+ "`registry_max_agent_age` flag.",
+ DEFAULT_REGISTRY_MAX_AGENT_COUNT);
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/9dad0b03/src/master/flags.hpp
----------------------------------------------------------------------
diff --git a/src/master/flags.hpp b/src/master/flags.hpp
index c6e8530..708a629 100644
--- a/src/master/flags.hpp
+++ b/src/master/flags.hpp
@@ -89,6 +89,9 @@ public:
size_t max_completed_tasks_per_framework;
Option<std::string> master_contender;
Option<std::string> master_detector;
+ Duration registry_gc_interval;
+ Duration registry_max_agent_age;
+ size_t registry_max_agent_count;
#ifdef WITH_NETWORK_ISOLATOR
Option<size_t> max_executors_per_agent;
http://git-wip-us.apache.org/repos/asf/mesos/blob/9dad0b03/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 71caea6..5089b99 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -1201,6 +1201,10 @@ void Master::finalize()
Clock::cancel(slaves.recoveredTimer.get());
}
+ if (registryGcTimer.isSome()) {
+ Clock::cancel(registryGcTimer.get());
+ }
+
terminate(whitelistWatcher);
wait(whitelistWatcher);
delete whitelistWatcher;
@@ -1574,6 +1578,9 @@ Future<Nothing> Master::_recover(const Registry& registry)
slaves.unreachable[unreachable.id()] = unreachable.timestamp();
}
+ // Set up a timer for age-based registry GC.
+ scheduleRegistryGc();
+
// Set up a timeout for slaves to re-register.
slaves.recoveredTimer =
delay(flags.agent_reregister_timeout,
@@ -1675,6 +1682,103 @@ Future<Nothing> Master::_recover(const Registry& registry)
}
+void Master::scheduleRegistryGc()
+{
+ registryGcTimer = delay(flags.registry_gc_interval,
+ self(),
+ &Self::doRegistryGc);
+}
+
+
+void Master::doRegistryGc()
+{
+ // Schedule next periodic GC.
+ scheduleRegistryGc();
+
+ // Determine which unreachable agents to GC from the registry, if
+ // any. We do this by examining the master's in-memory copy of the
+ // unreachable list and checking two criteria, "age" and "count". To
+ // check the "count" criteria, we remove elements from the beginning
+ // of the list until it contains at most "registry_max_agent_count"
+ // elements (note that `slaves.unreachable` is a `LinkedHashMap`,
+ // which provides iteration over keys in insertion-order). To check
+ // the "age" criteria, we remove any element in the list whose age
+ // is more than "registry_max_agent_age". Note that for the latter,
+ // we check the entire list, not just the beginning: this avoids
+ // requiring that the list be kept sorted by timestamp.
+ //
+ // We build a candidate list of SlaveIDs to remove. We then try to
+ // remove this list from the registry. Note that all the slaveIDs we
+ // want to remove might not be found in the registrar's copy of the
+ // unreachable list; this can occur if there is a concurrent write
+ // (e.g., an unreachable agent we want to GC reregisters
+ // concurrently). In this situation, we skip removing any elements
+ // we don't find.
+
+ size_t unreachableCount = slaves.unreachable.size();
+ TimeInfo currentTime = protobuf::getCurrentTime();
+ hashset<SlaveID> toRemove;
+
+ foreach (const SlaveID& slave, slaves.unreachable.keys()) {
+ // Count-based GC.
+ CHECK(toRemove.size() <= unreachableCount);
+
+ size_t liveCount = unreachableCount - toRemove.size();
+ if (liveCount > flags.registry_max_agent_count) {
+ toRemove.insert(slave);
+ continue;
+ }
+
+ // Age-based GC.
+ const TimeInfo& unreachableTime = slaves.unreachable[slave];
+ Duration age = Nanoseconds(
+ currentTime.nanoseconds() - unreachableTime.nanoseconds());
+
+ if (age > flags.registry_max_agent_age) {
+ toRemove.insert(slave);
+ }
+ }
+
+ if (toRemove.empty()) {
+ VLOG(1) << "Skipping periodic registry garbage collection: "
+ << "no agents qualify for removal";
+ return;
+ }
+
+ VLOG(1) << "Attempting to remove " << toRemove.size()
+ << " unreachable agents from the registry";
+
+ registrar->apply(Owned<Operation>(new PruneUnreachable(toRemove)))
+ .onAny(defer(self(),
+ &Self::_doRegistryGc,
+ toRemove,
+ lambda::_1));
+}
+
+
+void Master::_doRegistryGc(
+ const hashset<SlaveID>& toRemove,
+ const Future<bool>& registrarResult)
+{
+ CHECK(!registrarResult.isDiscarded());
+ CHECK(!registrarResult.isFailed());
+
+ // `PruneUnreachable` registry operation should never fail.
+ CHECK(registrarResult.get());
+
+ // TODO(neilc): Add a metric for # of agents discarded from the registry?
+ LOG(INFO) << "Garbage collected " << toRemove.size()
+ << " unreachable agents from the registry";
+
+ // Update in-memory state to be consistent with registry changes.
+ foreach (const SlaveID& slave, toRemove) {
+ // NOTE: `slave` might not appear in `slaves.unreachable` if there
+ // have been concurrent changes.
+ slaves.unreachable.erase(slave);
+ }
+}
+
+
void Master::recoveredSlavesTimeout(const Registry& registry)
{
CHECK(elected());
http://git-wip-us.apache.org/repos/asf/mesos/blob/9dad0b03/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index 370e0f0..4ba8e53 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -56,6 +56,7 @@
#include <stout/foreach.hpp>
#include <stout/hashmap.hpp>
#include <stout/hashset.hpp>
+#include <stout/linkedhashmap.hpp>
#include <stout/multihashmap.hpp>
#include <stout/option.hpp>
#include <stout/recordio.hpp>
@@ -964,6 +965,14 @@ private:
return leader.isSome() && leader.get() == info_;
}
+ void scheduleRegistryGc();
+
+ void doRegistryGc();
+
+ void _doRegistryGc(
+ const hashset<SlaveID>& toRemove,
+ const process::Future<bool>& registrarResult);
+
process::Future<bool> authorizeLogAccess(
const Option<std::string>& principal);
@@ -1595,6 +1604,10 @@ private:
// master is elected as a leader.
Option<process::Future<Nothing>> recovered;
+ // If this is the leading master, we periodically check whether we
+ // should GC some information from the registry.
+ Option<process::Timer> registryGcTimer;
+
struct Slaves
{
Slaves() : removed(MAX_REMOVED_SLAVES) {}
@@ -1696,10 +1709,13 @@ private:
// TODO(bmahler): Ideally we could use a cache with set semantics.
Cache<SlaveID, Nothing> removed;
- // Slaves that have been marked unreachable. We recover this from the
- // registry, so it includes slaves marked as unreachable by other
- // instances of the master.
- hashmap<SlaveID, TimeInfo> unreachable;
+ // Slaves that have been marked unreachable. We recover this from
+ // the registry, so it includes slaves marked as unreachable by
+ // other instances of the master. Note that we use a linkedhashmap
+ // to ensure the order of elements here matches the order in the
+ // registry's unreachable list, which matches the order in which
+ // agents are marked unreachable.
+ LinkedHashMap<SlaveID, TimeInfo> unreachable;
// This rate limiter is used to limit the removal of slaves failing
// health checks.
@@ -2051,6 +2067,48 @@ private:
};
+class PruneUnreachable : public Operation
+{
+public:
+ explicit PruneUnreachable(const hashset<SlaveID>& _toRemove)
+ : toRemove(_toRemove) {}
+
+protected:
+ virtual Try<bool> perform(Registry* registry, hashset<SlaveID>*, bool)
+ {
+ // Attempt to remove the SlaveIDs in `toRemove` from the
+ // unreachable list. Some SlaveIDs in `toRemove` might not appear
+ // in the registry; this is possible if there was a concurrent
+ // registry operation.
+ //
+ // TODO(neilc): This has quadratic worst-case behavior, because
+ // `DeleteSubrange` for a `repeated` object takes linear time.
+ bool mutate = false;
+ int i = 0;
+ while (i < registry->unreachable().slaves().size()) {
+ const Registry::UnreachableSlave& slave =
+ registry->unreachable().slaves(i);
+
+ if (toRemove.contains(slave.id())) {
+ Registry::UnreachableSlaves* unreachable =
+ registry->mutable_unreachable();
+
+ unreachable->mutable_slaves()->DeleteSubrange(i, i+1);
+ mutate = true;
+ continue;
+ }
+
+ i++;
+ }
+
+ return mutate;
+ }
+
+private:
+ const hashset<SlaveID> toRemove;
+};
+
+
// Implementation of slave removal Registrar operation.
class RemoveSlave : public Operation
{
http://git-wip-us.apache.org/repos/asf/mesos/blob/9dad0b03/src/master/registry.proto
----------------------------------------------------------------------
diff --git a/src/master/registry.proto b/src/master/registry.proto
index 03c896c..eab9821 100644
--- a/src/master/registry.proto
+++ b/src/master/registry.proto
@@ -76,6 +76,12 @@ message Registry {
// Slaves that have failed health checks. They may or may not still
// be running.
+ //
+ // New entries are added to the end of this list; hence the first
+ // element of the list was added first (although if there is clock
+ // drift, it might not necessarily have the smallest timestamp). The
+ // size of this list is limited by the `registry_max_agent_age` and
+ // `registry_max_agent_count` flags.
optional UnreachableSlaves unreachable = 7;
// Holds a list of machines and some status information about each.
http://git-wip-us.apache.org/repos/asf/mesos/blob/9dad0b03/src/tests/partition_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/partition_tests.cpp b/src/tests/partition_tests.cpp
index 997eda3..54d07b7 100644
--- a/src/tests/partition_tests.cpp
+++ b/src/tests/partition_tests.cpp
@@ -58,6 +58,7 @@ using process::Future;
using process::Message;
using process::Owned;
using process::PID;
+using process::Time;
using process::http::OK;
using process::http::Response;
@@ -1279,6 +1280,595 @@ TEST_P(PartitionTest, PartitionedSlaveExitedExecutor)
}
+// This test checks that the master correctly garbage collects
+// information about unreachable agents from the registry using the
+// count-based GC criterion.
+TEST_P(PartitionTest, RegistryGcByCount)
+{
+ // Configure GC to only keep the most recent partitioned agent in
+ // the unreachable list.
+ master::Flags masterFlags = CreateMasterFlags();
+ masterFlags.registry_strict = GetParam();
+ masterFlags.registry_max_agent_count = 1;
+
+ // Test logic assumes that two agents can be marked unreachable in
+ // sequence without triggering GC.
+ const Duration health_check_duration =
+ masterFlags.agent_ping_timeout * masterFlags.max_agent_ping_timeouts;
+
+ CHECK(masterFlags.registry_gc_interval >= health_check_duration * 2);
+
+ // Pause the clock before starting the master. This ensures that we
+ // know precisely when the GC timer will fire.
+ Clock::pause();
+
+ Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
+ ASSERT_SOME(master);
+
+ // Set these expectations up before we spawn the slave so that we
+ // don't miss the first PING.
+ Future<Message> ping = FUTURE_MESSAGE(
+ Eq(PingSlaveMessage().GetTypeName()), _, _);
+
+ // Drop all the PONGs to simulate slave partition.
+ DROP_PROTOBUFS(PongSlaveMessage(), _, _);
+
+ Future<SlaveRegisteredMessage> slaveRegisteredMessage1 =
+ FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _);
+
+ Owned<MasterDetector> slaveDetector1 = master.get()->createDetector();
+ Try<Owned<cluster::Slave>> slave1 = StartSlave(slaveDetector1.get());
+ ASSERT_SOME(slave1);
+
+ AWAIT_READY(slaveRegisteredMessage1);
+ const SlaveID slaveId1 = slaveRegisteredMessage1.get().slave_id();
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(&driver, _, _));
+
+ Future<Nothing> resourceOffers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureSatisfy(&resourceOffers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ // Need to make sure the framework AND slave have registered with
+ // master. Waiting for resource offers should accomplish both.
+ AWAIT_READY(resourceOffers);
+
+ EXPECT_CALL(sched, offerRescinded(&driver, _))
+ .WillRepeatedly(Return());
+
+ Future<Nothing> slaveLost1;
+ EXPECT_CALL(sched, slaveLost(&driver, _))
+ .WillOnce(FutureSatisfy(&slaveLost1));
+
+ // Simulate the first slave becoming partitioned from the master.
+ size_t pings = 0;
+ while (true) {
+ AWAIT_READY(ping);
+ pings++;
+ if (pings == masterFlags.max_agent_ping_timeouts) {
+ break;
+ }
+ ping = FUTURE_MESSAGE(Eq(PingSlaveMessage().GetTypeName()), _, _);
+ Clock::advance(masterFlags.agent_ping_timeout);
+ }
+
+ Clock::advance(masterFlags.agent_ping_timeout);
+ Clock::settle();
+
+ // Record the time at which we expect the master to have marked the
+ // agent as unreachable. We then advance the clock -- this shouldn't
+ // do anything, but it ensures that the `unreachable_time` we check
+ // below is computed at the right time.
+ TimeInfo partitionTime1 = protobuf::getCurrentTime();
+
+ Clock::advance(Milliseconds(100));
+
+ AWAIT_READY(slaveLost1);
+
+ // Shutdown the first slave. This is necessary because we only drop
+ // PONG messages; after advancing the clock below, the slave would
+ // try to reregister and would succeed. Hence, stop the slave first.
+ slave1.get()->terminate();
+ slave1->reset();
+
+ // Start another slave.
+ Future<SlaveRegisteredMessage> slaveRegisteredMessage2 =
+ FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _);
+
+ Owned<MasterDetector> slaveDetector2 = master.get()->createDetector();
+ Try<Owned<cluster::Slave>> slave2 = StartSlave(slaveDetector2.get());
+ ASSERT_SOME(slave2);
+
+ AWAIT_READY(slaveRegisteredMessage2);
+ const SlaveID slaveId2 = slaveRegisteredMessage2.get().slave_id();
+
+ Future<Nothing> slaveLost2;
+ EXPECT_CALL(sched, slaveLost(&driver, _))
+ .WillOnce(FutureSatisfy(&slaveLost2));
+
+ // Simulate the second slave becoming partitioned from the master.
+ pings = 0;
+ while (true) {
+ AWAIT_READY(ping);
+ pings++;
+ if (pings == masterFlags.max_agent_ping_timeouts) {
+ break;
+ }
+ ping = FUTURE_MESSAGE(Eq(PingSlaveMessage().GetTypeName()), _, _);
+ Clock::advance(masterFlags.agent_ping_timeout);
+ }
+
+ Clock::advance(masterFlags.agent_ping_timeout);
+ Clock::settle();
+
+ // Record the time at which we expect the master to have marked the
+ // agent as unreachable. We then advance the clock -- this shouldn't
+ // do anything, but it ensures that the `unreachable_time` we check
+ // below is computed at the right time.
+ TimeInfo partitionTime2 = protobuf::getCurrentTime();
+
+ Clock::advance(Milliseconds(100));
+
+ AWAIT_READY(slaveLost2);
+
+ // Shutdown the second slave. This is necessary because we only drop
+ // PONG messages; after advancing the clock below, the slave would
+ // try to reregister and would succeed. Hence, stop the slave first.
+ slave2.get()->terminate();
+ slave2->reset();
+
+ // Do explicit reconciliation for a random task ID on `slave1`. GC
+ // has not occurred yet (since `registry_gc_interval` has not
+ // elapsed since the master was started), so the slave should be in
+ // the unreachable list; hence `unreachable_time` should be set on
+ // the result of the reconciliation request.
+ TaskStatus status1;
+ status1.mutable_task_id()->set_value(UUID::random().toString());
+ status1.mutable_slave_id()->CopyFrom(slaveId1);
+ status1.set_state(TASK_STAGING); // Dummy value.
+
+ Future<TaskStatus> reconcileUpdate1;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&reconcileUpdate1));
+
+ driver.reconcileTasks({status1});
+
+ AWAIT_READY(reconcileUpdate1);
+ EXPECT_EQ(TASK_LOST, reconcileUpdate1.get().state());
+ EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate1.get().reason());
+ EXPECT_EQ(partitionTime1, reconcileUpdate1.get().unreachable_time());
+
+ // Advance the clock to cause GC to be performed.
+ Clock::advance(masterFlags.registry_gc_interval);
+ Clock::settle();
+
+ // Do explicit reconciliation for a random task ID on `slave1`.
+ // Because the agent has been removed from the unreachable list in
+ // the registry, `unreachable_time` should NOT be set.
+ TaskStatus status2;
+ status2.mutable_task_id()->set_value(UUID::random().toString());
+ status2.mutable_slave_id()->CopyFrom(slaveId1);
+ status2.set_state(TASK_STAGING); // Dummy value.
+
+ Future<TaskStatus> reconcileUpdate2;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&reconcileUpdate2));
+
+ driver.reconcileTasks({status2});
+
+ AWAIT_READY(reconcileUpdate2);
+ EXPECT_EQ(TASK_LOST, reconcileUpdate2.get().state());
+ EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate2.get().reason());
+ EXPECT_FALSE(reconcileUpdate2.get().has_unreachable_time());
+
+ // Do explicit reconciliation for a random task ID on the second
+ // partitioned slave. Because the agent is still in the unreachable
+ // list in the registry, `unreachable_time` should be set.
+ TaskStatus status3;
+ status3.mutable_task_id()->set_value(UUID::random().toString());
+ status3.mutable_slave_id()->CopyFrom(slaveId2);
+ status3.set_state(TASK_STAGING); // Dummy value.
+
+ Future<TaskStatus> reconcileUpdate3;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&reconcileUpdate3));
+
+ driver.reconcileTasks({status3});
+
+ AWAIT_READY(reconcileUpdate3);
+ EXPECT_EQ(TASK_LOST, reconcileUpdate3.get().state());
+ EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate3.get().reason());
+ EXPECT_EQ(partitionTime2, reconcileUpdate3.get().unreachable_time());
+
+ JSON::Object stats = Metrics();
+ EXPECT_EQ(2, stats.values["master/slave_unreachable_scheduled"]);
+ EXPECT_EQ(2, stats.values["master/slave_unreachable_completed"]);
+ EXPECT_EQ(2, stats.values["master/slave_removals"]);
+ EXPECT_EQ(2, stats.values["master/slave_removals/reason_unhealthy"]);
+
+ driver.stop();
+ driver.join();
+
+ Clock::resume();
+}
+
+
+// This test ensures that when using the count-based criterion to
+// garbage collect unreachable agents from the registry, the agents
+// that were marked unreachable first are the ones that are
+// removed. This requires creating a large unreachable list, which
+// would be annoying to do by creating slaves and simulating network
+// partitions; instead we add agents to the unreachable list by
+// directly applying registry operations.
+TEST_P(PartitionTest, RegistryGcByCountManySlaves)
+{
+ // Configure GC to only keep the most recent partitioned agent in
+ // the unreachable list.
+ master::Flags masterFlags = CreateMasterFlags();
+ masterFlags.registry_strict = GetParam();
+ masterFlags.registry_max_agent_count = 1;
+
+ Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
+ ASSERT_SOME(master);
+
+ Clock::pause();
+
+ TimeInfo unreachableTime = protobuf::getCurrentTime();
+
+ vector<SlaveID> slaveIDs;
+ for (int i = 0; i < 50; i++) {
+ SlaveID slaveID;
+ slaveID.set_value(UUID::random().toString());
+ slaveIDs.push_back(slaveID);
+
+ SlaveInfo slaveInfo;
+ slaveInfo.set_hostname("localhost");
+ slaveInfo.mutable_id()->CopyFrom(slaveID);
+
+ Future<bool> admitApply =
+ master.get()->registrar->unmocked_apply(
+ Owned<master::Operation>(
+ new master::AdmitSlave(slaveInfo)));
+
+ AWAIT_EXPECT_TRUE(admitApply);
+
+ Future<bool> unreachableApply =
+ master.get()->registrar->unmocked_apply(
+ Owned<master::Operation>(
+ new master::MarkSlaveUnreachable(slaveInfo, unreachableTime)));
+
+ AWAIT_EXPECT_TRUE(unreachableApply);
+ }
+
+ // Restart the master to recover from updated registry.
+ master->reset();
+ master = StartMaster(masterFlags);
+ ASSERT_SOME(master);
+
+ // Advance the clock to cause GC to be performed.
+ Clock::advance(masterFlags.registry_gc_interval);
+ Clock::settle();
+
+ // Start a scheduler: we verify GC behavior by doing reconciliation.
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ Future<Nothing> registered;
+ EXPECT_CALL(sched, registered(&driver, _, _))
+ .WillOnce(FutureSatisfy(®istered));
+
+ driver.start();
+
+ AWAIT_READY(registered);
+
+ // We expect that only the most-recently inserted SlaveID will have
+ // survived GC. We also check that the second-most-recent SlaveID
+ // has been GC'd.
+ SlaveID keptSlaveID = slaveIDs.back();
+ SlaveID removedSlaveID = *(slaveIDs.crbegin() + 1);
+
+ TaskStatus status1;
+ status1.mutable_task_id()->set_value(UUID::random().toString());
+ status1.mutable_slave_id()->CopyFrom(keptSlaveID);
+ status1.set_state(TASK_STAGING); // Dummy value.
+
+ Future<TaskStatus> reconcileUpdate1;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&reconcileUpdate1));
+
+ driver.reconcileTasks({status1});
+
+ AWAIT_READY(reconcileUpdate1);
+ EXPECT_EQ(TASK_LOST, reconcileUpdate1.get().state());
+ EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate1.get().reason());
+ EXPECT_EQ(unreachableTime, reconcileUpdate1.get().unreachable_time());
+
+ TaskStatus status2;
+ status2.mutable_task_id()->set_value(UUID::random().toString());
+ status2.mutable_slave_id()->CopyFrom(removedSlaveID);
+ status2.set_state(TASK_STAGING); // Dummy value.
+
+ Future<TaskStatus> reconcileUpdate2;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&reconcileUpdate2));
+
+ driver.reconcileTasks({status2});
+
+ AWAIT_READY(reconcileUpdate2);
+ EXPECT_EQ(TASK_LOST, reconcileUpdate2.get().state());
+ EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate2.get().reason());
+ EXPECT_FALSE(reconcileUpdate2.get().has_unreachable_time());
+
+ driver.stop();
+ driver.join();
+
+ Clock::resume();
+}
+
+
+// This test checks that the master correctly garbage collects
+// information about unreachable agents from the registry using the
+// age-based GC criterion. We configure GC to discard agents after 20
+// minutes; GC occurs every 15 mins. We test the following schedule:
+//
+// 75 sec: slave1 is marked unreachable
+// 720 secs (12 mins): slave2 is marked unreachable
+// 900 secs (15 mins): GC runs, nothing discarded
+// 1800 secs (30 mins): GC runs, slave1 is discarded
+// 2700 secs (45 mins): GC runs, slave2 is discarded
+TEST_P(PartitionTest, RegistryGcByAge)
+{
+ master::Flags masterFlags = CreateMasterFlags();
+ masterFlags.registry_gc_interval = Minutes(15);
+ masterFlags.registry_max_agent_age = Minutes(20);
+ masterFlags.registry_strict = GetParam();
+
+ // Pause the clock before starting the master. This ensures that we
+ // know precisely when the GC timer will fire.
+ Clock::pause();
+
+ Time startTime = Clock::now();
+
+ Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
+ ASSERT_SOME(master);
+
+ // Set these expectations up before we spawn the slave so that we
+ // don't miss the first PING.
+ Future<Message> ping = FUTURE_MESSAGE(
+ Eq(PingSlaveMessage().GetTypeName()), _, _);
+
+ // Drop all the PONGs to simulate slave partition.
+ DROP_PROTOBUFS(PongSlaveMessage(), _, _);
+
+ Future<SlaveRegisteredMessage> slaveRegisteredMessage1 =
+ FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _);
+
+ Owned<MasterDetector> slaveDetector1 = master.get()->createDetector();
+ Try<Owned<cluster::Slave>> slave1 = StartSlave(slaveDetector1.get());
+ ASSERT_SOME(slave1);
+
+ AWAIT_READY(slaveRegisteredMessage1);
+ const SlaveID slaveId1 = slaveRegisteredMessage1.get().slave_id();
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(&driver, _, _));
+
+ Future<Nothing> resourceOffers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureSatisfy(&resourceOffers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ // Need to make sure the framework AND slave have registered with
+ // master. Waiting for resource offers should accomplish both.
+ AWAIT_READY(resourceOffers);
+
+ EXPECT_CALL(sched, offerRescinded(&driver, _))
+ .WillRepeatedly(Return());
+
+ Future<Nothing> slaveLost1;
+ EXPECT_CALL(sched, slaveLost(&driver, _))
+ .WillOnce(FutureSatisfy(&slaveLost1));
+
+ // Simulate the first slave becoming partitioned from the master.
+ size_t pings = 0;
+ while (true) {
+ AWAIT_READY(ping);
+ pings++;
+ if (pings == masterFlags.max_agent_ping_timeouts) {
+ break;
+ }
+ ping = FUTURE_MESSAGE(Eq(PingSlaveMessage().GetTypeName()), _, _);
+ Clock::advance(masterFlags.agent_ping_timeout);
+ }
+
+ Clock::advance(masterFlags.agent_ping_timeout);
+ Clock::settle();
+
+ // Record the time at which we expect the master to have marked the
+ // agent as unreachable.
+ TimeInfo partitionTime1 = protobuf::getCurrentTime();
+
+ AWAIT_READY(slaveLost1);
+
+ // Shutdown the first slave. This is necessary because we only drop
+ // PONG messages; after advancing the clock below, the slave would
+ // try to reregister and would succeed. Hence, stop the slave first.
+ slave1.get()->terminate();
+ slave1->reset();
+
+ // Per the schedule above, we want the second slave to be
+ // partitioned after 720 seconds have elapsed, so we advance the
+ // clock by 570 seconds (570 + 75 + 75 = 720).
+ EXPECT_EQ(Seconds(75), Clock::now() - startTime);
+
+ Clock::advance(Seconds(570));
+
+ // Start another slave.
+ Future<SlaveRegisteredMessage> slaveRegisteredMessage2 =
+ FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _);
+
+ Owned<MasterDetector> slaveDetector2 = master.get()->createDetector();
+ Try<Owned<cluster::Slave>> slave2 = StartSlave(slaveDetector2.get());
+ ASSERT_SOME(slave2);
+
+ AWAIT_READY(slaveRegisteredMessage2);
+ const SlaveID slaveId2 = slaveRegisteredMessage2.get().slave_id();
+
+ Future<Nothing> slaveLost2;
+ EXPECT_CALL(sched, slaveLost(&driver, _))
+ .WillOnce(FutureSatisfy(&slaveLost2));
+
+ // Simulate the second slave becoming partitioned from the master.
+ pings = 0;
+ while (true) {
+ AWAIT_READY(ping);
+ pings++;
+ if (pings == masterFlags.max_agent_ping_timeouts) {
+ break;
+ }
+ ping = FUTURE_MESSAGE(Eq(PingSlaveMessage().GetTypeName()), _, _);
+ Clock::advance(masterFlags.agent_ping_timeout);
+ }
+
+ Clock::advance(masterFlags.agent_ping_timeout);
+ Clock::settle();
+
+ // Record the time at which we expect the master to have marked the
+ // agent as unreachable.
+ TimeInfo partitionTime2 = protobuf::getCurrentTime();
+
+ AWAIT_READY(slaveLost2);
+
+ // Shutdown the second slave. This is necessary because we only drop
+ // PONG messages; after advancing the clock below, the slave would
+ // try to reregister and would succeed. Hence, stop the slave first.
+ slave2.get()->terminate();
+ slave2->reset();
+
+ EXPECT_EQ(Seconds(720), Clock::now() - startTime);
+
+ // Advance the clock to trigger a GC. The first GC occurs at 900
+ // elapsed seconds, so we advance by 180 seconds.
+ Clock::advance(Seconds(180));
+ Clock::settle();
+
+ // Do explicit reconciliation for random task IDs on both slaves.
+ // Since neither slave has exceeded the age-based GC bound, we
+ // expect to find both slaves (i.e., `unreachable_time` will be set).
+ TaskStatus status1;
+ status1.mutable_task_id()->set_value(UUID::random().toString());
+ status1.mutable_slave_id()->CopyFrom(slaveId1);
+ status1.set_state(TASK_STAGING); // Dummy value.
+
+ Future<TaskStatus> reconcileUpdate1;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&reconcileUpdate1));
+
+ driver.reconcileTasks({status1});
+
+ AWAIT_READY(reconcileUpdate1);
+ EXPECT_EQ(TASK_LOST, reconcileUpdate1.get().state());
+ EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate1.get().reason());
+ EXPECT_EQ(partitionTime1, reconcileUpdate1.get().unreachable_time());
+
+ TaskStatus status2;
+ status2.mutable_task_id()->set_value(UUID::random().toString());
+ status2.mutable_slave_id()->CopyFrom(slaveId2);
+ status2.set_state(TASK_STAGING); // Dummy value.
+
+ Future<TaskStatus> reconcileUpdate2;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&reconcileUpdate2));
+
+ driver.reconcileTasks({status2});
+
+ AWAIT_READY(reconcileUpdate2);
+ EXPECT_EQ(TASK_LOST, reconcileUpdate2.get().state());
+ EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate2.get().reason());
+ EXPECT_EQ(partitionTime2, reconcileUpdate2.get().unreachable_time());
+
+ // Advance the clock to cause GC to be performed.
+ Clock::advance(Minutes(15));
+ Clock::settle();
+
+ // Do explicit reconciliation for random task IDs on both slaves.
+ // We expect `slave1` to have been garbage collected, but `slave2`
+ // should still be present in the registry.
+ TaskStatus status3;
+ status3.mutable_task_id()->set_value(UUID::random().toString());
+ status3.mutable_slave_id()->CopyFrom(slaveId1);
+ status3.set_state(TASK_STAGING); // Dummy value.
+
+ Future<TaskStatus> reconcileUpdate3;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&reconcileUpdate3));
+
+ driver.reconcileTasks({status3});
+
+ AWAIT_READY(reconcileUpdate3);
+ EXPECT_EQ(TASK_LOST, reconcileUpdate3.get().state());
+ EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate3.get().reason());
+ EXPECT_FALSE(reconcileUpdate3.get().has_unreachable_time());
+
+ TaskStatus status4;
+ status4.mutable_task_id()->set_value(UUID::random().toString());
+ status4.mutable_slave_id()->CopyFrom(slaveId2);
+ status4.set_state(TASK_STAGING); // Dummy value.
+
+ Future<TaskStatus> reconcileUpdate4;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&reconcileUpdate4));
+
+ driver.reconcileTasks({status4});
+
+ AWAIT_READY(reconcileUpdate4);
+ EXPECT_EQ(TASK_LOST, reconcileUpdate4.get().state());
+ EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate4.get().reason());
+ EXPECT_EQ(partitionTime2, reconcileUpdate4.get().unreachable_time());
+
+ // Advance the clock to cause GC to be performed.
+ Clock::advance(Minutes(15));
+ Clock::settle();
+
+ // Do explicit reconciliation for a random task ID on `slave2`. We
+ // expect that it has been garbage collected, which means
+ // `unreachable_time` will not be set.
+ TaskStatus status5;
+ status5.mutable_task_id()->set_value(UUID::random().toString());
+ status5.mutable_slave_id()->CopyFrom(slaveId2);
+ status5.set_state(TASK_STAGING); // Dummy value.
+
+ Future<TaskStatus> reconcileUpdate5;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&reconcileUpdate5));
+
+ driver.reconcileTasks({status5});
+
+ AWAIT_READY(reconcileUpdate5);
+ EXPECT_EQ(TASK_LOST, reconcileUpdate5.get().state());
+ EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate5.get().reason());
+ EXPECT_FALSE(reconcileUpdate5.get().has_unreachable_time());
+
+ driver.stop();
+ driver.join();
+
+ Clock::resume();
+}
+
+
class OneWayPartitionTest : public MesosTest {};
http://git-wip-us.apache.org/repos/asf/mesos/blob/9dad0b03/src/tests/registrar_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/registrar_tests.cpp b/src/tests/registrar_tests.cpp
index 7ea00df..745cded 100644
--- a/src/tests/registrar_tests.cpp
+++ b/src/tests/registrar_tests.cpp
@@ -358,6 +358,52 @@ TEST_P(RegistrarTest, MarkUnreachable)
}
+TEST_P(RegistrarTest, PruneUnreachable)
+{
+ Registrar registrar(flags, state);
+ AWAIT_READY(registrar.recover(master));
+
+ SlaveInfo info1;
+ info1.set_hostname("localhost");
+
+ SlaveID id1;
+ id1.set_value("1");
+ info1.mutable_id()->CopyFrom(id1);
+
+ SlaveID id2;
+ id2.set_value("2");
+
+ SlaveInfo info2;
+ info2.set_hostname("localhost");
+ info2.mutable_id()->CopyFrom(id2);
+
+ AWAIT_TRUE(registrar.apply(Owned<Operation>(new AdmitSlave(info1))));
+ AWAIT_TRUE(registrar.apply(Owned<Operation>(new AdmitSlave(info2))));
+
+ AWAIT_TRUE(
+ registrar.apply(
+ Owned<Operation>(
+ new MarkSlaveUnreachable(info1, protobuf::getCurrentTime()))));
+
+ AWAIT_TRUE(
+ registrar.apply(
+ Owned<Operation>(
+ new MarkSlaveUnreachable(info2, protobuf::getCurrentTime()))));
+
+ AWAIT_TRUE(registrar.apply(Owned<Operation>(new PruneUnreachable({id1}))));
+ AWAIT_TRUE(registrar.apply(Owned<Operation>(new PruneUnreachable({id2}))));
+
+ AWAIT_TRUE(registrar.apply(Owned<Operation>(new AdmitSlave(info1))));
+
+ AWAIT_TRUE(
+ registrar.apply(
+ Owned<Operation>(
+ new MarkSlaveUnreachable(info1, protobuf::getCurrentTime()))));
+
+ AWAIT_TRUE(registrar.apply(Owned<Operation>(new PruneUnreachable({id1}))));
+}
+
+
TEST_P(RegistrarTest, Remove)
{
Registrar registrar(flags, state);
[06/27] mesos git commit: Added more assertions to the master.
Posted by vi...@apache.org.
Added more assertions to the master.
Review: https://reviews.apache.org/r/50846/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/3cecdbdd
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/3cecdbdd
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/3cecdbdd
Branch: refs/heads/master
Commit: 3cecdbdd4b727567c621f999fb8df45116f0a979
Parents: ac26919
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:47:33 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:47:33 2016 -0700
----------------------------------------------------------------------
src/master/master.cpp | 11 +++++++++--
src/master/master.hpp | 3 ---
2 files changed, 9 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/3cecdbdd/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index d5a194b..34de462 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -5183,6 +5183,9 @@ void Master::_reregisterSlave(
// example.
bool slaveWasRemoved = slaves.removed.get(slave->id).isSome();
+ slaves.removed.erase(slave->id);
+ slaves.unreachable.erase(slave->id);
+
addSlave(slave, completedFrameworks);
Duration pingTimeout =
@@ -5673,6 +5676,9 @@ void Master::markUnreachable(const SlaveID& slaveId)
LOG(INFO) << "Marking agent " << *slave
<< " unreachable: health check timed out";
+ CHECK(!slaves.unreachable.contains(slaveId));
+ CHECK(slaves.removed.get(slaveId).isNone());
+
// We want to remove the slave first, to avoid the allocator
// re-allocating the recovered resources.
//
@@ -7036,9 +7042,10 @@ void Master::addSlave(
const vector<Archive::Framework>& completedFrameworks)
{
CHECK_NOTNULL(slave);
+ CHECK(!slaves.registered.contains(slave->id));
+ CHECK(!slaves.unreachable.contains(slave->id));
+ CHECK(slaves.removed.get(slave->id).isNone());
- slaves.removed.erase(slave->id);
- slaves.unreachable.erase(slave->id);
slaves.registered.put(slave);
link(slave->pid);
http://git-wip-us.apache.org/repos/asf/mesos/blob/3cecdbdd/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index 6aeb1b9..7e50359 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -1928,9 +1928,6 @@ protected:
}
}
- // TODO(neilc): Check if the slave appears in the list of
- // `unreachable` slaves in the registry?
-
Registry::Slave* slave = registry->mutable_slaves()->add_slaves();
slave->mutable_info()->CopyFrom(info);
slaveIDs->insert(info.id());
[18/27] mesos git commit: Improved
SlaveTest.HealthCheckUnregisterRace.
Posted by vi...@apache.org.
Improved SlaveTest.HealthCheckUnregisterRace.
Now that the MockRegistrar has been introduced, we can assert
that the master doesn't do a registry operation in certain
situations.
Review: https://reviews.apache.org/r/51845/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/6172da93
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/6172da93
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/6172da93
Branch: refs/heads/master
Commit: 6172da93d883cf2bafb7cf195ad119d480434329
Parents: fac32e4
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:48:48 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:48:48 2016 -0700
----------------------------------------------------------------------
src/tests/slave_tests.cpp | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/6172da93/src/tests/slave_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/slave_tests.cpp b/src/tests/slave_tests.cpp
index 8ad3547..552f3f8 100644
--- a/src/tests/slave_tests.cpp
+++ b/src/tests/slave_tests.cpp
@@ -2875,7 +2875,12 @@ TEST_F(SlaveTest, HealthCheckUnregisterRace)
// We now want to arrange for the agent to fail health checks. We
// can't do that directly, because the `SlaveObserver` for this
// agent has already been removed. Instead, we dispatch to the
- // master's `markUnreachable` method directly.
+ // master's `markUnreachable` method directly. We expect the master
+ // to ignore this message; in particular, the master should not
+ // attempt to update the registry to mark the slave unreachable.
+ EXPECT_CALL(*master.get()->registrar.get(), apply(_))
+ .Times(0);
+
process::dispatch(master.get()->pid, &Master::markUnreachable, slaveId);
Clock::settle();
[12/27] mesos git commit: Added test for HTTP endpoints during slave
removal.
Posted by vi...@apache.org.
Added test for HTTP endpoints during slave removal.
When a slave is being removed but before the registry operation to
remove it has completed, querying the HTTP endpoints at the master
should not indicate that the slave has been removed yet. This is
important because the registry update might fail; if the master
fails over, the new master will not have removed the slave.
Review: https://reviews.apache.org/r/51377/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/592da1ed
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/592da1ed
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/592da1ed
Branch: refs/heads/master
Commit: 592da1edc2a5279f8851a8f30a1425229e8fbaca
Parents: 87b9431
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:48:11 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:48:11 2016 -0700
----------------------------------------------------------------------
src/tests/master_tests.cpp | 117 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 117 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/592da1ed/src/tests/master_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/master_tests.cpp b/src/tests/master_tests.cpp
index 737f589..6c49ab3 100644
--- a/src/tests/master_tests.cpp
+++ b/src/tests/master_tests.cpp
@@ -607,6 +607,123 @@ TEST_F(MasterTest, KillUnknownTaskSlaveInTransition)
}
+// This test checks that the HTTP endpoints return the expected
+// information for agents that the master is in the process of marking
+// unreachable, but that have not yet been so marked (because the
+// registry update hasn't completed yet).
+TEST_F(MasterTest, EndpointsForHalfRemovedSlave)
+{
+ master::Flags masterFlags = CreateMasterFlags();
+ Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
+ ASSERT_SOME(master);
+
+ // Set these expectations up before we spawn the slave so that we
+ // don't miss the first PING.
+ Future<process::Message> ping = FUTURE_MESSAGE(
+ Eq(PingSlaveMessage().GetTypeName()), _, _);
+
+ // Drop all the PONGs to simulate slave partition.
+ DROP_PROTOBUFS(PongSlaveMessage(), _, _);
+
+ Owned<MasterDetector> detector = master.get()->createDetector();
+ Try<Owned<cluster::Slave>> slave = StartSlave(detector.get());
+ ASSERT_SOME(slave);
+
+ Clock::pause();
+
+ // Now advance through the PINGs.
+ size_t pings = 0;
+ while (true) {
+ AWAIT_READY(ping);
+ pings++;
+ if (pings == masterFlags.max_agent_ping_timeouts) {
+ break;
+ }
+ ping = FUTURE_MESSAGE(Eq(PingSlaveMessage().GetTypeName()), _, _);
+ Clock::advance(masterFlags.agent_ping_timeout);
+ }
+
+ // Intercept the first registrar operation that is attempted; this
+ // should be the operation that marks the slave as unreachable.
+ Future<Owned<master::Operation>> unreachable;
+ Promise<bool> promise;
+ EXPECT_CALL(*master.get()->registrar.get(), apply(_))
+ .WillOnce(DoAll(FutureArg<0>(&unreachable),
+ Return(promise.future())));
+
+ Clock::advance(masterFlags.agent_ping_timeout);
+
+ slave.get()->terminate();
+ slave->reset();
+
+ // Wait for the master to attempt to update the registry, but don't
+ // allow the registry update to succeed yet.
+ AWAIT_READY(unreachable);
+ EXPECT_NE(
+ nullptr,
+ dynamic_cast<master::MarkSlaveUnreachable*>(unreachable.get().get()));
+
+ // Settle the clock for the sake of paranoia.
+ Clock::settle();
+
+ // Metrics should not be updated yet.
+ JSON::Object stats1 = Metrics();
+ EXPECT_EQ(1, stats1.values["master/slave_unreachable_scheduled"]);
+ EXPECT_EQ(1, stats1.values["master/slave_unreachable_completed"]);
+ EXPECT_EQ(0, stats1.values["master/slave_removals"]);
+ EXPECT_EQ(0, stats1.values["master/slave_removals/reason_unhealthy"]);
+ EXPECT_EQ(0, stats1.values["master/slave_removals/reason_unregistered"]);
+
+ // HTTP endpoints (e.g., /state) should not reflect the removal of
+ // the slave yet.
+ Future<Response> response1 = process::http::get(
+ master.get()->pid,
+ "state",
+ None(),
+ createBasicAuthHeaders(DEFAULT_CREDENTIAL));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response1);
+ AWAIT_EXPECT_RESPONSE_HEADER_EQ(APPLICATION_JSON, "Content-Type", response1);
+
+ Try<JSON::Object> parse1 = JSON::parse<JSON::Object>(response1.get().body);
+ Result<JSON::Array> array1 = parse1.get().find<JSON::Array>("slaves");
+ ASSERT_SOME(array1);
+ EXPECT_EQ(1u, array1.get().values.size());
+
+ // Allow the registry operation to return success. Note that we
+ // don't actually update the registry here, since the test doesn't
+ // require it.
+ promise.set(true);
+
+ Clock::settle();
+
+ // Metrics should be updated.
+ JSON::Object stats2 = Metrics();
+ EXPECT_EQ(1, stats2.values["master/slave_unreachable_scheduled"]);
+ EXPECT_EQ(1, stats2.values["master/slave_unreachable_completed"]);
+ EXPECT_EQ(1, stats2.values["master/slave_removals"]);
+ EXPECT_EQ(1, stats2.values["master/slave_removals/reason_unhealthy"]);
+ EXPECT_EQ(0, stats2.values["master/slave_removals/reason_unregistered"]);
+
+ // HTTP endpoints should be updated.
+ Future<Response> response2 = process::http::get(
+ master.get()->pid,
+ "state",
+ None(),
+ createBasicAuthHeaders(DEFAULT_CREDENTIAL));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response2);
+ AWAIT_EXPECT_RESPONSE_HEADER_EQ(APPLICATION_JSON, "Content-Type", response2);
+
+ Try<JSON::Object> parse2 = JSON::parse<JSON::Object>(response2.get().body);
+ Result<JSON::Array> array2 = parse2.get().find<JSON::Array>("slaves");
+ ASSERT_SOME(array2);
+ EXPECT_EQ(0u, array2.get().values.size());
+
+ Clock::resume();
+}
+
+
TEST_F(MasterTest, StatusUpdateAck)
{
Try<Owned<cluster::Master>> master = StartMaster();
[25/27] mesos git commit: Squelched GMock warning in
MasterTest.RecoveredSlaveCanReregister.
Posted by vi...@apache.org.
Squelched GMock warning in MasterTest.RecoveredSlaveCanReregister.
After the slave reregisters, the framework might receive an offer for
the resources on that slave.
Review: https://reviews.apache.org/r/51956/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/89a78b87
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/89a78b87
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/89a78b87
Branch: refs/heads/master
Commit: 89a78b873f5751714fefc7e97e1249e094d1e68a
Parents: 905204e
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:49:29 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:49:29 2016 -0700
----------------------------------------------------------------------
src/tests/master_tests.cpp | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/89a78b87/src/tests/master_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/master_tests.cpp b/src/tests/master_tests.cpp
index a32ac12..e6c8362 100644
--- a/src/tests/master_tests.cpp
+++ b/src/tests/master_tests.cpp
@@ -1956,11 +1956,17 @@ TEST_F(MasterTest, RecoveredSlaveCanReregister)
Future<SlaveReregisteredMessage> slaveReregisteredMessage =
FUTURE_PROTOBUF(SlaveReregisteredMessage(), master.get()->pid, _);
+ // Expect a resource offer from the re-registered slave.
+ Future<Nothing> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureSatisfy(&offers));
+
detector = master.get()->createDetector();
slave = StartSlave(detector.get(), slaveFlags);
ASSERT_SOME(slave);
AWAIT_READY(slaveReregisteredMessage);
+ AWAIT_READY(offers);
driver.stop();
driver.join();
[04/27] mesos git commit: Fixed outdated comment.
Posted by vi...@apache.org.
Fixed outdated comment.
Review: https://reviews.apache.org/r/50844/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/06506fae
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/06506fae
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/06506fae
Branch: refs/heads/master
Commit: 06506faea2b9ead713907e8dd19ab53505dc93cf
Parents: 504e05e
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:47:19 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:47:19 2016 -0700
----------------------------------------------------------------------
src/master/master.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/06506fae/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 7e501ec..89fde24 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -7630,8 +7630,8 @@ InverseOffer* Master::getInverseOffer(const OfferID& inverseOfferId)
// Create a new framework ID. We format the ID as MASTERID-FWID, where
-// MASTERID is the ID of the master (launch date plus fault tolerant ID)
-// and FWID is an increasing integer.
+// MASTERID is the ID of the master (randomly generated UUID) and FWID
+// is an increasing integer.
FrameworkID Master::newFrameworkId()
{
std::ostringstream out;
[10/27] mesos git commit: Introduced MockRegistrar.
Posted by vi...@apache.org.
Introduced MockRegistrar.
This allows test cases to inspect how the master interacts with
the registry.
This commit changes `tests::cluster::Master` to use `MockRegistrar`
exclusively, even for test cases that aren't interested in mocking
aspects of the master <-> registrar interaction. However, this should be
harmless, since by default `MockRegistrar` behaves identically to the
normal registrar.
Review: https://reviews.apache.org/r/51375/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/80993da4
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/80993da4
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/80993da4
Branch: refs/heads/master
Commit: 80993da4cefa88a60a63134fadbc2a3f183dfd46
Parents: 88b2533
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:48:02 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:48:02 2016 -0700
----------------------------------------------------------------------
src/Makefile.am | 2 ++
src/master/registrar.hpp | 4 +--
src/tests/cluster.cpp | 3 +-
src/tests/cluster.hpp | 17 ++++++++--
src/tests/mock_registrar.cpp | 71 +++++++++++++++++++++++++++++++++++++++
src/tests/mock_registrar.hpp | 60 +++++++++++++++++++++++++++++++++
6 files changed, 151 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/80993da4/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 6fb095f..478fb5a 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -991,6 +991,7 @@ libmesos_no_3rdparty_la_SOURCES += \
tests/mesos.hpp \
tests/mock_docker.hpp \
tests/mock_slave.hpp \
+ tests/mock_registrar.hpp \
tests/module.hpp \
tests/script.hpp \
tests/utils.hpp \
@@ -2107,6 +2108,7 @@ mesos_tests_SOURCES = \
tests/metrics_tests.cpp \
tests/mock_docker.cpp \
tests/mock_slave.cpp \
+ tests/mock_registrar.cpp \
tests/module.cpp \
tests/module_tests.cpp \
tests/oversubscription_tests.cpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/80993da4/src/master/registrar.hpp
----------------------------------------------------------------------
diff --git a/src/master/registrar.hpp b/src/master/registrar.hpp
index c39dd1b..238a960 100644
--- a/src/master/registrar.hpp
+++ b/src/master/registrar.hpp
@@ -106,7 +106,7 @@ public:
Registrar(const Flags& flags,
mesos::state::protobuf::State* state,
const Option<std::string>& authenticationRealm = None());
- ~Registrar();
+ virtual ~Registrar();
// Recovers the Registry, persisting the new Master information.
// The Registrar must be recovered to allow other operations to
@@ -123,7 +123,7 @@ public:
// false if the operation is not permitted.
// Failure if the operation fails (possibly lost log leadership),
// or recovery failed.
- process::Future<bool> apply(process::Owned<Operation> operation);
+ virtual process::Future<bool> apply(process::Owned<Operation> operation);
// Gets the pid of the underlying process.
// Used in tests.
http://git-wip-us.apache.org/repos/asf/mesos/blob/80993da4/src/tests/cluster.cpp
----------------------------------------------------------------------
diff --git a/src/tests/cluster.cpp b/src/tests/cluster.cpp
index b04653a..7152ac3 100644
--- a/src/tests/cluster.cpp
+++ b/src/tests/cluster.cpp
@@ -88,6 +88,7 @@
#include "slave/containerizer/fetcher.hpp"
#include "tests/cluster.hpp"
+#include "tests/mock_registrar.hpp"
using mesos::master::contender::StandaloneMasterContender;
using mesos::master::contender::ZooKeeperMasterContender;
@@ -243,7 +244,7 @@ Try<process::Owned<Master>> Master::start(
// Instantiate some other master dependencies.
master->state.reset(new mesos::state::protobuf::State(master->storage.get()));
- master->registrar.reset(new master::Registrar(
+ master->registrar.reset(new MockRegistrar(
flags, master->state.get(), master::READONLY_HTTP_AUTHENTICATION_REALM));
if (slaveRemovalLimiter.isNone() && flags.agent_removal_rate_limit.isSome()) {
http://git-wip-us.apache.org/repos/asf/mesos/blob/80993da4/src/tests/cluster.hpp
----------------------------------------------------------------------
diff --git a/src/tests/cluster.hpp b/src/tests/cluster.hpp
index c6fbbf2..250b12f 100644
--- a/src/tests/cluster.hpp
+++ b/src/tests/cluster.hpp
@@ -53,7 +53,6 @@
#include "master/constants.hpp"
#include "master/flags.hpp"
#include "master/master.hpp"
-#include "master/registrar.hpp"
#include "slave/constants.hpp"
#include "slave/flags.hpp"
@@ -64,6 +63,7 @@
#include "slave/containerizer/containerizer.hpp"
#include "slave/containerizer/fetcher.hpp"
+#include "tests/mock_registrar.hpp"
namespace mesos {
namespace internal {
@@ -111,7 +111,12 @@ private:
Option<zookeeper::URL> zookeeperUrl;
Files files;
- // Dependencies that are created by the factory method.
+ // Dependencies that are created by the factory method. The order in
+ // which these fields are declared should match the order in which
+ // they are initialized by the constructor; this ensures that
+ // dependencies between these fields are handled correctly during
+ // destruction.
+
process::Owned<mesos::allocator::Allocator> allocator;
process::Owned<Authorizer> authorizer;
process::Owned<mesos::master::contender::MasterContender> contender;
@@ -119,8 +124,14 @@ private:
process::Owned<mesos::log::Log> log;
process::Owned<mesos::state::Storage> storage;
process::Owned<mesos::state::protobuf::State> state;
- process::Owned<master::Registrar> registrar;
+public:
+ // Exposed for testing and mocking purposes. We always use a
+ // `MockRegistrar` in case the test case wants to inspect how the
+ // master interacts with the registrar; by default, the mock
+ // registrar behaves identically to the normal registrar.
+ process::Owned<MockRegistrar> registrar;
+private:
Option<std::shared_ptr<process::RateLimiter>> slaveRemovalLimiter;
// Indicates whether or not authorization callbacks were set when this master
http://git-wip-us.apache.org/repos/asf/mesos/blob/80993da4/src/tests/mock_registrar.cpp
----------------------------------------------------------------------
diff --git a/src/tests/mock_registrar.cpp b/src/tests/mock_registrar.cpp
new file mode 100644
index 0000000..8643e4c
--- /dev/null
+++ b/src/tests/mock_registrar.cpp
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+
+#include <gmock/gmock.h>
+
+#include <mesos/state/protobuf.hpp>
+
+#include <process/future.hpp>
+#include <process/owned.hpp>
+
+#include <stout/none.hpp>
+#include <stout/option.hpp>
+
+#include "master/flags.hpp"
+#include "master/registrar.hpp"
+
+#include "tests/mock_registrar.hpp"
+
+using std::string;
+
+using testing::_;
+using testing::DoDefault;
+using testing::Invoke;
+
+using process::Future;
+using process::Owned;
+
+namespace mesos {
+namespace internal {
+namespace tests {
+
+MockRegistrar::MockRegistrar(
+ const master::Flags& flags,
+ mesos::state::protobuf::State* state,
+ const Option<string>& authenticationRealm)
+ : Registrar(flags, state, authenticationRealm)
+{
+ // The default behavior is to call the original method.
+ ON_CALL(*this, apply(_))
+ .WillByDefault(Invoke(this, &MockRegistrar::unmocked_apply));
+ EXPECT_CALL(*this, apply(_))
+ .WillRepeatedly(DoDefault());
+}
+
+
+MockRegistrar::~MockRegistrar() {}
+
+
+Future<bool> MockRegistrar::unmocked_apply(Owned<master::Operation> operation)
+{
+ return master::Registrar::apply(operation);
+}
+
+} // namespace tests {
+} // namespace internal {
+} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/80993da4/src/tests/mock_registrar.hpp
----------------------------------------------------------------------
diff --git a/src/tests/mock_registrar.hpp b/src/tests/mock_registrar.hpp
new file mode 100644
index 0000000..cdcc699
--- /dev/null
+++ b/src/tests/mock_registrar.hpp
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __TESTS_MOCK_REGISTRAR_HPP__
+#define __TESTS_MOCK_REGISTRAR_HPP__
+
+#include <string>
+
+#include <gmock/gmock.h>
+
+#include <mesos/state/protobuf.hpp>
+
+#include <process/future.hpp>
+#include <process/owned.hpp>
+
+#include <stout/none.hpp>
+#include <stout/option.hpp>
+
+#include "master/flags.hpp"
+#include "master/registrar.hpp"
+
+namespace mesos {
+namespace internal {
+namespace tests {
+
+class MockRegistrar : public mesos::internal::master::Registrar
+{
+public:
+ MockRegistrar(const master::Flags& flags,
+ mesos::state::protobuf::State* state,
+ const Option<std::string>& authenticationRealm = None());
+
+ virtual ~MockRegistrar();
+
+ MOCK_METHOD1(
+ apply,
+ process::Future<bool>(process::Owned<master::Operation> operation));
+
+ process::Future<bool> unmocked_apply(
+ process::Owned<master::Operation> operation);
+};
+
+} // namespace tests {
+} // namespace internal {
+} // namespace mesos {
+
+#endif // __TESTS_MOCK_REGISTRAR_HPP__
[27/27] mesos git commit: Removed use of "--registry_strict" master
flag.
Posted by vi...@apache.org.
Removed use of "--registry_strict" master flag.
The flag will remain for a few more months per the deprecation policy,
but we no longer need to set it in the unit tests. Providing an
extensive usage summary in "--help" output also doesn't seem useful.
Review: https://reviews.apache.org/r/51958/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/29236068
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/29236068
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/29236068
Branch: refs/heads/master
Commit: 29236068f23b6cfbd19d7a4b5b96be852818a356
Parents: 6280299
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:49:38 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:49:38 2016 -0700
----------------------------------------------------------------------
src/master/flags.cpp | 5 +----
src/tests/cluster.cpp | 6 ++----
src/tests/mesos.cpp | 1 -
3 files changed, 3 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/29236068/src/master/flags.cpp
----------------------------------------------------------------------
diff --git a/src/master/flags.cpp b/src/master/flags.cpp
index aea8bd4..9bfb40e 100644
--- a/src/master/flags.cpp
+++ b/src/master/flags.cpp
@@ -89,10 +89,7 @@ mesos::internal::master::Flags::Flags()
add(&Flags::registry_strict,
"registry_strict",
"Whether the master will take actions based on the persistent\n"
- "information stored in the Registry. Setting this to false means\n"
- "that the Registrar will never reject the admission, readmission,\n"
- "or removal of an agent. Consequently, `false` can be used to\n"
- "bootstrap the persistent state on a running cluster.\n"
+ "information stored in the Registry.\n"
"NOTE: This flag is *disabled* and will be removed in a future\n"
"version of Mesos.",
false,
http://git-wip-us.apache.org/repos/asf/mesos/blob/29236068/src/tests/cluster.cpp
----------------------------------------------------------------------
diff --git a/src/tests/cluster.cpp b/src/tests/cluster.cpp
index 7152ac3..f201b49 100644
--- a/src/tests/cluster.cpp
+++ b/src/tests/cluster.cpp
@@ -192,10 +192,8 @@ Try<process::Owned<Master>> Master::start(
}
// Check for some invalid flag combinations.
- if (flags.registry == "in_memory" && flags.registry_strict) {
- return Error(
- "Cannot use '--registry_strict' when using in-memory storage based"
- " registry");
+ if (flags.registry_strict) {
+ return Error("Support for '--registry_strict' has been removed");
}
if (flags.registry == "replicated_log" && flags.work_dir.isNone()) {
http://git-wip-us.apache.org/repos/asf/mesos/blob/29236068/src/tests/mesos.cpp
----------------------------------------------------------------------
diff --git a/src/tests/mesos.cpp b/src/tests/mesos.cpp
index 07a64f0..2aae160 100644
--- a/src/tests/mesos.cpp
+++ b/src/tests/mesos.cpp
@@ -145,7 +145,6 @@ master::Flags MesosTest::CreateMasterFlags()
// Use the replicated log (without ZooKeeper) by default.
flags.registry = "replicated_log";
- flags.registry_strict = true;
// On many test VMs, this default is too small.
flags.registry_store_timeout = flags.registry_store_timeout * 5;
[03/27] mesos git commit: Added test for race between health checks
and agent unregistration.
Posted by vi...@apache.org.
Added test for race between health checks and agent unregistration.
This test tries to simulate a race between marking an agent unhealthy
and receiving an `UnregisterSlaveMessage` for that agent.
Unfortunately, this test is a little fragile (we need to manually
dispatch an event to the master process to simulate the action that
would be taken by the slave observer).
Review: https://reviews.apache.org/r/50707/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/504e05ee
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/504e05ee
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/504e05ee
Branch: refs/heads/master
Commit: 504e05ee60c3648a4bf2d7433f1380731db8999f
Parents: 9ba79da
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:47:14 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:47:14 2016 -0700
----------------------------------------------------------------------
src/tests/slave_tests.cpp | 72 ++++++++++++++++++++++++++++++++++++++++++
1 file changed, 72 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/504e05ee/src/tests/slave_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/slave_tests.cpp b/src/tests/slave_tests.cpp
index 696e0f7..2e7accc 100644
--- a/src/tests/slave_tests.cpp
+++ b/src/tests/slave_tests.cpp
@@ -2813,6 +2813,78 @@ TEST_F(SlaveTest, CancelSlaveRemoval)
}
+// This test checks that the master behaves correctly when a slave
+// fails health checks, but concurrently the slave unregisters from
+// the master.
+TEST_F(SlaveTest, HealthCheckUnregisterRace)
+{
+ // Start a master.
+ Try<Owned<cluster::Master>> master = StartMaster();
+ ASSERT_SOME(master);
+
+ // Start a slave.
+ Owned<MasterDetector> detector = master.get()->createDetector();
+ Try<Owned<cluster::Slave>> slave = StartSlave(detector.get());
+ ASSERT_SOME(slave);
+
+ // Start a scheduler.
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(&driver, _, _));
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ // Need to make sure the framework AND slave have registered with
+ // master. Waiting for resource offers should accomplish both.
+ AWAIT_READY(offers);
+
+ SlaveID slaveId = offers.get()[0].slave_id();
+
+ EXPECT_CALL(sched, offerRescinded(&driver, _))
+ .Times(1); // Expect a single offer to be rescinded.
+
+ Future<Nothing> slaveLost;
+ EXPECT_CALL(sched, slaveLost(&driver, _))
+ .WillOnce(FutureSatisfy(&slaveLost));
+
+ // Cause the slave to shutdown gracefully by sending it SIGUSR1.
+ // This should result in the slave sending `UnregisterSlaveMessage`
+ // to the master.
+ Future<UnregisterSlaveMessage> unregisterSlaveMessage =
+ FUTURE_PROTOBUF(
+ UnregisterSlaveMessage(),
+ slave.get()->pid,
+ master.get()->pid);
+
+ kill(getpid(), SIGUSR1);
+
+ AWAIT_READY(unregisterSlaveMessage);
+ AWAIT_READY(slaveLost);
+
+ Clock::pause();
+ Clock::settle();
+
+ // We now want to arrange for the agent to fail health checks. We
+ // can't do that directly, because the `SlaveObserver` for this
+ // agent has already been removed. Instead, we dispatch to the
+ // master's `markUnreachable` method directly.
+ process::dispatch(master.get()->pid, &Master::markUnreachable, slaveId);
+
+ Clock::settle();
+ Clock::resume();
+
+ driver.stop();
+ driver.join();
+}
+
+
// This test ensures that a killTask() can happen between runTask()
// and _run() and then gets "handled properly". This means that
// the task never gets started, but also does not get lost. The end
[08/27] mesos git commit: Fixed flakiness in
MasterAuthorizationTest.SlaveDisconnected.
Posted by vi...@apache.org.
Fixed flakiness in MasterAuthorizationTest.SlaveDisconnected.
The goal of this test is to check this interleaving:
task launch: block on authorization
slave terminates gracefully: removed by master
auth succeeds: we expect the task launch to fail
However, the test neglected to ensure that slave removal was completed
before allowing authorization to succeed.
Review: https://reviews.apache.org/r/51371/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/0cb24804
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/0cb24804
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/0cb24804
Branch: refs/heads/master
Commit: 0cb248044c23ff702a1099e724474fe5c1a5c17e
Parents: df037de
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:47:48 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:47:48 2016 -0700
----------------------------------------------------------------------
src/tests/master_authorization_tests.cpp | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/0cb24804/src/tests/master_authorization_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/master_authorization_tests.cpp b/src/tests/master_authorization_tests.cpp
index a6399f1..74422c7 100644
--- a/src/tests/master_authorization_tests.cpp
+++ b/src/tests/master_authorization_tests.cpp
@@ -660,18 +660,17 @@ TEST_F(MasterAuthorizationTest, SlaveDisconnected)
// Wait until authorization is in progress.
AWAIT_READY(authorize);
+ Future<Nothing> slaveLost;
EXPECT_CALL(sched, slaveLost(&driver, _))
- .Times(AtMost(1));
-
- Future<Nothing> deactivateSlave =
- FUTURE_DISPATCH(_, &MesosAllocatorProcess::deactivateSlave);
+ .WillOnce(FutureSatisfy(&slaveLost));
- // Stop the slave with explicit shutdown message so that the master does not
- // wait for it to reconnect.
+ // Stop the slave with explicit shutdown message so that the master
+ // does not wait for it to reconnect.
slave.get()->shutdown();
slave->reset();
- AWAIT_READY(deactivateSlave);
+ // Wait for the slave to be removed by the master.
+ AWAIT_READY(slaveLost);
Future<TaskStatus> status;
EXPECT_CALL(sched, statusUpdate(&driver, _))
[26/27] mesos git commit: Removed "strictness" parameter from
RegistrarTest and PartitionTest.
Posted by vi...@apache.org.
Removed "strictness" parameter from RegistrarTest and PartitionTest.
This parameter should no longer result in any difference in behavior.
Review: https://reviews.apache.org/r/51957/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/6280299b
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/6280299b
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/6280299b
Branch: refs/heads/master
Commit: 6280299bf78f42929daecd271cbdad4e6ce24f62
Parents: 89a78b8
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:49:34 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:49:34 2016 -0700
----------------------------------------------------------------------
src/tests/partition_tests.cpp | 55 ++++++++++----------------------------
src/tests/registrar_tests.cpp | 49 +++++++++++++--------------------
2 files changed, 32 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/6280299b/src/tests/partition_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/partition_tests.cpp b/src/tests/partition_tests.cpp
index 895a073..7c38f0e 100644
--- a/src/tests/partition_tests.cpp
+++ b/src/tests/partition_tests.cpp
@@ -79,21 +79,14 @@ namespace internal {
namespace tests {
-class PartitionTest : public MesosTest,
- public WithParamInterface<bool> {};
-
-
-// The Registrar tests are parameterized by "strictness".
-INSTANTIATE_TEST_CASE_P(Strict, PartitionTest, ::testing::Bool());
+class PartitionTest : public MesosTest {};
// This test checks that a scheduler gets a slave lost
// message for a partitioned slave.
-TEST_P(PartitionTest, PartitionedSlave)
+TEST_F(PartitionTest, PartitionedSlave)
{
master::Flags masterFlags = CreateMasterFlags();
- masterFlags.registry_strict = GetParam();
-
Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
ASSERT_SOME(master);
@@ -171,13 +164,11 @@ TEST_P(PartitionTest, PartitionedSlave)
// This test checks that a slave can reregister with the master after
// a partition, and that PARTITION_AWARE tasks running on the slave
// continue to run.
-TEST_P(PartitionTest, ReregisterSlavePartitionAware)
+TEST_F(PartitionTest, ReregisterSlavePartitionAware)
{
Clock::pause();
master::Flags masterFlags = CreateMasterFlags();
- masterFlags.registry_strict = GetParam();
-
Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
ASSERT_SOME(master);
@@ -317,13 +308,11 @@ TEST_P(PartitionTest, ReregisterSlavePartitionAware)
// This test checks that a slave can reregister with the master after
// a partition, and that non-PARTITION_AWARE tasks running on the
// slave are shutdown.
-TEST_P(PartitionTest, ReregisterSlaveNotPartitionAware)
+TEST_F(PartitionTest, ReregisterSlaveNotPartitionAware)
{
Clock::pause();
master::Flags masterFlags = CreateMasterFlags();
- masterFlags.registry_strict = GetParam();
-
Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
ASSERT_SOME(master);
@@ -480,13 +469,11 @@ TEST_P(PartitionTest, ReregisterSlaveNotPartitionAware)
// not. Both tasks should survive the reregistration of the partitioned
// agent: we allow the non-partition-aware task to continue running for
// backward compatibility with the "non-strict" Mesos 1.0 behavior.
-TEST_P(PartitionTest, PartitionedSlaveReregistrationMasterFailover)
+TEST_F(PartitionTest, PartitionedSlaveReregistrationMasterFailover)
{
Clock::pause();
master::Flags masterFlags = CreateMasterFlags();
- masterFlags.registry_strict = GetParam();
-
Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
ASSERT_SOME(master);
@@ -733,13 +720,11 @@ TEST_P(PartitionTest, PartitionedSlaveReregistrationMasterFailover)
// a task for a PARTITION_AWARE scheduler. The scheduler disconnects
// before the partition heals. Right now, the task is left running as
// an orphan; once MESOS-4659 is fixed, the task should be shutdown.
-TEST_P(PartitionTest, PartitionedSlaveOrphanedTask)
+TEST_F(PartitionTest, PartitionedSlaveOrphanedTask)
{
Clock::pause();
master::Flags masterFlags = CreateMasterFlags();
- masterFlags.registry_strict = GetParam();
-
Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
ASSERT_SOME(master);
@@ -903,13 +888,11 @@ TEST_P(PartitionTest, PartitionedSlaveOrphanedTask)
// master (e.g., because of a spurious Zk leader flag at the slave),
// the master does not kill any tasks on the slave, even if those
// tasks are not PARTITION_AWARE.
-TEST_P(PartitionTest, SpuriousSlaveReregistration)
+TEST_F(PartitionTest, SpuriousSlaveReregistration)
{
Clock::pause();
master::Flags masterFlags = CreateMasterFlags();
- masterFlags.registry_strict = GetParam();
-
Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
ASSERT_SOME(master);
@@ -1006,13 +989,11 @@ TEST_P(PartitionTest, SpuriousSlaveReregistration)
// master's POV). In prior Mesos versions, the master would shutdown
// the slave in this situation. In Mesos >= 1.1, the master will drop
// the status update; the slave will eventually try to reregister.
-TEST_P(PartitionTest, PartitionedSlaveStatusUpdates)
+TEST_F(PartitionTest, PartitionedSlaveStatusUpdates)
{
Clock::pause();
master::Flags masterFlags = CreateMasterFlags();
- masterFlags.registry_strict = GetParam();
-
Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
ASSERT_SOME(master);
@@ -1160,11 +1141,9 @@ TEST_P(PartitionTest, PartitionedSlaveStatusUpdates)
// versions, the master would shutdown the slave in this situation. In
// Mesos >= 1.1, the master will drop the message; the slave will
// eventually try to reregister.
-TEST_P(PartitionTest, PartitionedSlaveExitedExecutor)
+TEST_F(PartitionTest, PartitionedSlaveExitedExecutor)
{
master::Flags masterFlags = CreateMasterFlags();
- masterFlags.registry_strict = GetParam();
-
Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
ASSERT_SOME(master);
@@ -1282,12 +1261,11 @@ TEST_P(PartitionTest, PartitionedSlaveExitedExecutor)
// This test checks that the master correctly garbage collects
// information about unreachable agents from the registry using the
// count-based GC criterion.
-TEST_P(PartitionTest, RegistryGcByCount)
+TEST_F(PartitionTest, RegistryGcByCount)
{
// Configure GC to only keep the most recent partitioned agent in
// the unreachable list.
master::Flags masterFlags = CreateMasterFlags();
- masterFlags.registry_strict = GetParam();
masterFlags.registry_max_agent_count = 1;
// Test logic assumes that two agents can be marked unreachable in
@@ -1510,12 +1488,11 @@ TEST_P(PartitionTest, RegistryGcByCount)
// would be annoying to do by creating slaves and simulating network
// partitions; instead we add agents to the unreachable list by
// directly applying registry operations.
-TEST_P(PartitionTest, RegistryGcByCountManySlaves)
+TEST_F(PartitionTest, RegistryGcByCountManySlaves)
{
// Configure GC to only keep the most recent partitioned agent in
// the unreachable list.
master::Flags masterFlags = CreateMasterFlags();
- masterFlags.registry_strict = GetParam();
masterFlags.registry_max_agent_count = 1;
Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
@@ -1632,12 +1609,11 @@ TEST_P(PartitionTest, RegistryGcByCountManySlaves)
// 900 secs (15 mins): GC runs, nothing discarded
// 1800 secs (30 mins): GC runs, slave1 is discarded
// 2700 secs (45 mins): GC runs, slave2 is discarded
-TEST_P(PartitionTest, RegistryGcByAge)
+TEST_F(PartitionTest, RegistryGcByAge)
{
master::Flags masterFlags = CreateMasterFlags();
masterFlags.registry_gc_interval = Minutes(15);
masterFlags.registry_max_agent_age = Minutes(20);
- masterFlags.registry_strict = GetParam();
// Pause the clock before starting the master. This ensures that we
// know precisely when the GC timer will fire.
@@ -1888,10 +1864,9 @@ TEST_P(PartitionTest, RegistryGcByAge)
// configure GC to only keep a single agent. Concurrently with GC
// running, we arrange for one of those agents to reregister with the
// master.
-TEST_P(PartitionTest, RegistryGcRace2)
+TEST_F(PartitionTest, RegistryGcRace2)
{
master::Flags masterFlags = CreateMasterFlags();
- masterFlags.registry_strict = GetParam();
masterFlags.registry_max_agent_count = 1;
Clock::pause();
@@ -2165,11 +2140,9 @@ TEST_P(PartitionTest, RegistryGcRace2)
// health checks twice. At present, this can only occur if the registry
// operation to mark the slave unreachable takes so long that the
// slave fails an additional health check in the mean time.
-TEST_P(PartitionTest, FailHealthChecksTwice)
+TEST_F(PartitionTest, FailHealthChecksTwice)
{
master::Flags masterFlags = CreateMasterFlags();
- masterFlags.registry_strict = GetParam();
-
Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
ASSERT_SOME(master);
http://git-wip-us.apache.org/repos/asf/mesos/blob/6280299b/src/tests/registrar_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/registrar_tests.cpp b/src/tests/registrar_tests.cpp
index 6c42fd1..0433b26 100644
--- a/src/tests/registrar_tests.cpp
+++ b/src/tests/registrar_tests.cpp
@@ -212,23 +212,10 @@ protected:
};
-class RegistrarTest : public RegistrarTestBase,
- public WithParamInterface<bool>
-{
-protected:
- virtual void SetUp()
- {
- RegistrarTestBase::SetUp();
- flags.registry_strict = GetParam();
- }
-};
-
-
-// The Registrar tests are parameterized by "strictness".
-INSTANTIATE_TEST_CASE_P(Strict, RegistrarTest, ::testing::Bool());
+class RegistrarTest : public RegistrarTestBase {};
-TEST_P(RegistrarTest, Recover)
+TEST_F(RegistrarTest, Recover)
{
Registrar registrar(flags, state);
@@ -268,7 +255,7 @@ TEST_P(RegistrarTest, Recover)
}
-TEST_P(RegistrarTest, Admit)
+TEST_F(RegistrarTest, Admit)
{
Registrar registrar(flags, state);
AWAIT_READY(registrar.recover(master));
@@ -278,7 +265,7 @@ TEST_P(RegistrarTest, Admit)
}
-TEST_P(RegistrarTest, MarkReachable)
+TEST_F(RegistrarTest, MarkReachable)
{
Registrar registrar(flags, state);
AWAIT_READY(registrar.recover(master));
@@ -310,7 +297,7 @@ TEST_P(RegistrarTest, MarkReachable)
}
-TEST_P(RegistrarTest, MarkUnreachable)
+TEST_F(RegistrarTest, MarkUnreachable)
{
Registrar registrar(flags, state);
AWAIT_READY(registrar.recover(master));
@@ -353,7 +340,7 @@ TEST_P(RegistrarTest, MarkUnreachable)
}
-TEST_P(RegistrarTest, PruneUnreachable)
+TEST_F(RegistrarTest, PruneUnreachable)
{
Registrar registrar(flags, state);
AWAIT_READY(registrar.recover(master));
@@ -399,7 +386,7 @@ TEST_P(RegistrarTest, PruneUnreachable)
}
-TEST_P(RegistrarTest, Remove)
+TEST_F(RegistrarTest, Remove)
{
Registrar registrar(flags, state);
AWAIT_READY(registrar.recover(master));
@@ -460,7 +447,7 @@ TEST_P(RegistrarTest, Remove)
// Adds maintenance schedules to the registry, one machine at a time.
// Then removes machines from the schedule.
-TEST_P(RegistrarTest, UpdateMaintenanceSchedule)
+TEST_F(RegistrarTest, UpdateMaintenanceSchedule)
{
// Machine definitions used in this test.
MachineID machine1;
@@ -619,7 +606,7 @@ TEST_P(RegistrarTest, UpdateMaintenanceSchedule)
// Creates a schedule and properly starts maintenance.
-TEST_P(RegistrarTest, StartMaintenance)
+TEST_F(RegistrarTest, StartMaintenance)
{
// Machine definitions used in this test.
MachineID machine1;
@@ -706,7 +693,7 @@ TEST_P(RegistrarTest, StartMaintenance)
// Creates a schedule and properly starts and stops maintenance.
-TEST_P(RegistrarTest, StopMaintenance)
+TEST_F(RegistrarTest, StopMaintenance)
{
// Machine definitions used in this test.
MachineID machine1;
@@ -788,7 +775,7 @@ TEST_P(RegistrarTest, StopMaintenance)
// Tests that adding and updating quotas in the registry works properly.
-TEST_P(RegistrarTest, UpdateQuota)
+TEST_F(RegistrarTest, UpdateQuota)
{
const string ROLE1 = "role1";
const string ROLE2 = "role2";
@@ -915,7 +902,7 @@ TEST_P(RegistrarTest, UpdateQuota)
// Tests removing quotas from the registry.
-TEST_P(RegistrarTest, RemoveQuota)
+TEST_F(RegistrarTest, RemoveQuota)
{
const string ROLE1 = "role1";
const string ROLE2 = "role2";
@@ -993,7 +980,7 @@ TEST_P(RegistrarTest, RemoveQuota)
// Tests that updating weights in the registry works properly.
-TEST_P(RegistrarTest, UpdateWeights)
+TEST_F(RegistrarTest, UpdateWeights)
{
const string ROLE1 = "role1";
double WEIGHT1 = 2.0;
@@ -1056,7 +1043,7 @@ TEST_P(RegistrarTest, UpdateWeights)
}
-TEST_P(RegistrarTest, Bootstrap)
+TEST_F(RegistrarTest, Bootstrap)
{
// Run 1 simulates the reregistration of a slave that is not present
// in the registry.
@@ -1092,7 +1079,7 @@ public:
};
-TEST_P(RegistrarTest, FetchTimeout)
+TEST_F(RegistrarTest, FetchTimeout)
{
Clock::pause();
@@ -1121,7 +1108,7 @@ TEST_P(RegistrarTest, FetchTimeout)
}
-TEST_P(RegistrarTest, StoreTimeout)
+TEST_F(RegistrarTest, StoreTimeout)
{
Clock::pause();
@@ -1153,7 +1140,7 @@ TEST_P(RegistrarTest, StoreTimeout)
}
-TEST_P(RegistrarTest, Abort)
+TEST_F(RegistrarTest, Abort)
{
MockStorage storage;
State state(&storage);
@@ -1180,7 +1167,7 @@ TEST_P(RegistrarTest, Abort)
// Tests that requests to the '/registry' endpoint are authenticated when HTTP
// authentication is enabled.
-TEST_P(RegistrarTest, Authentication)
+TEST_F(RegistrarTest, Authentication)
{
const string AUTHENTICATION_REALM = "realm";
[11/27] mesos git commit: Adjusted existing tests to use
MockRegistrar.
Posted by vi...@apache.org.
Adjusted existing tests to use MockRegistrar.
This means that `Master::_reregisterSlave` can be made private.
Review: https://reviews.apache.org/r/51376/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/87b94314
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/87b94314
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/87b94314
Branch: refs/heads/master
Commit: 87b94314b1490ef7035e9ec5e8e071dfc1f12309
Parents: 80993da
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:48:06 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:48:06 2016 -0700
----------------------------------------------------------------------
src/master/master.hpp | 27 +++++++++++----------------
src/tests/master_tests.cpp | 16 ++++++++++++----
src/tests/reconciliation_tests.cpp | 21 ++++++++++++++-------
3 files changed, 37 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/87b94314/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index 714aa79..370e0f0 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -555,22 +555,6 @@ public:
// Made public for testing purposes.
process::Future<Nothing> _recover(const Registry& registry);
- // Continuation of reregisterSlave().
- // Made public for testing purposes.
- // TODO(vinod): Instead of doing this create and use a
- // MockRegistrar.
- // TODO(dhamon): Consider FRIEND_TEST macro from gtest.
- void _reregisterSlave(
- const SlaveInfo& slaveInfo,
- const process::UPID& pid,
- const std::vector<Resource>& checkpointedResources,
- const std::vector<ExecutorInfo>& executorInfos,
- const std::vector<Task>& tasks,
- const std::vector<FrameworkInfo>& frameworks,
- const std::vector<Archive::Framework>& completedFrameworks,
- const std::string& version,
- const process::Future<bool>& readmit);
-
MasterInfo info() const
{
return info_;
@@ -620,6 +604,17 @@ protected:
const std::string& version,
const process::Future<bool>& admit);
+ void _reregisterSlave(
+ const SlaveInfo& slaveInfo,
+ const process::UPID& pid,
+ const std::vector<Resource>& checkpointedResources,
+ const std::vector<ExecutorInfo>& executorInfos,
+ const std::vector<Task>& tasks,
+ const std::vector<FrameworkInfo>& frameworks,
+ const std::vector<Archive::Framework>& completedFrameworks,
+ const std::string& version,
+ const process::Future<bool>& readmit);
+
void __reregisterSlave(
Slave* slave,
const std::vector<Task>& tasks,
http://git-wip-us.apache.org/repos/asf/mesos/blob/87b94314/src/tests/master_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/master_tests.cpp b/src/tests/master_tests.cpp
index 85ebd57..737f589 100644
--- a/src/tests/master_tests.cpp
+++ b/src/tests/master_tests.cpp
@@ -535,9 +535,6 @@ TEST_F(MasterTest, KillUnknownTaskSlaveInTransition)
EXPECT_CALL(exec, shutdown(_))
.Times(AtMost(1));
- Future<Nothing> _reregisterSlave =
- DROP_DISPATCH(_, &Master::_reregisterSlave);
-
// Stop master and slave.
master->reset();
slave.get()->terminate();
@@ -551,6 +548,14 @@ TEST_F(MasterTest, KillUnknownTaskSlaveInTransition)
master = StartMaster();
ASSERT_SOME(master);
+ // Intercept the first registrar operation that is attempted; this
+ // should be the registry operation that reregisters the slave.
+ Future<Owned<master::Operation>> reregister;
+ Promise<bool> promise; // Never satisfied.
+ EXPECT_CALL(*master.get()->registrar.get(), apply(_))
+ .WillOnce(DoAll(FutureArg<0>(&reregister),
+ Return(promise.future())));
+
frameworkId = Future<FrameworkID>();
EXPECT_CALL(sched, registered(&driver, _, _))
.WillOnce(FutureArg<1>(&frameworkId));
@@ -567,7 +572,10 @@ TEST_F(MasterTest, KillUnknownTaskSlaveInTransition)
ASSERT_SOME(slave);
// Wait for the slave to start reregistration.
- AWAIT_READY(_reregisterSlave);
+ AWAIT_READY(reregister);
+ EXPECT_NE(
+ nullptr,
+ dynamic_cast<master::MarkSlaveReachable*>(reregister.get().get()));
// As Master::killTask isn't doing anything, we shouldn't get a status update.
EXPECT_CALL(sched, statusUpdate(&driver, _))
http://git-wip-us.apache.org/repos/asf/mesos/blob/87b94314/src/tests/reconciliation_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/reconciliation_tests.cpp b/src/tests/reconciliation_tests.cpp
index a043356..9828ab9 100644
--- a/src/tests/reconciliation_tests.cpp
+++ b/src/tests/reconciliation_tests.cpp
@@ -28,6 +28,7 @@
#include <process/clock.hpp>
#include <process/future.hpp>
+#include <process/gmock.hpp>
#include <process/owned.hpp>
#include <process/pid.hpp>
#include <process/process.hpp>
@@ -412,15 +413,18 @@ TEST_F(ReconciliationTest, SlaveInTransition)
EXPECT_CALL(sched, statusUpdate(&driver, _))
.Times(0);
- // Drop '&Master::_reregisterSlave' dispatch so that the slave is
- // in 'reregistering' state.
- Future<Nothing> _reregisterSlave =
- DROP_DISPATCH(_, &Master::_reregisterSlave);
-
// Restart the master.
master = StartMaster(masterFlags);
ASSERT_SOME(master);
+ // Intercept the first registrar operation that is attempted; this
+ // should be the registry operation that reregisters the slave.
+ Future<Owned<master::Operation>> reregister;
+ Promise<bool> promise; // Never satisfied.
+ EXPECT_CALL(*master.get()->registrar.get(), apply(_))
+ .WillOnce(DoAll(FutureArg<0>(&reregister),
+ Return(promise.future())));
+
driver.start();
// Wait for the framework to register.
@@ -431,8 +435,11 @@ TEST_F(ReconciliationTest, SlaveInTransition)
slave = StartSlave(detector.get(), slaveFlags);
ASSERT_SOME(slave);
- // Slave will be in 'reregistering' state here.
- AWAIT_READY(_reregisterSlave);
+ // Wait for the slave to start reregistration.
+ AWAIT_READY(reregister);
+ EXPECT_NE(
+ nullptr,
+ dynamic_cast<master::MarkSlaveReachable*>(reregister.get().get()));
Future<mesos::scheduler::Call> reconcileCall = FUTURE_CALL(
mesos::scheduler::Call(), mesos::scheduler::Call::RECONCILE, _ , _);
[02/27] mesos git commit: Added test cases for PARTITION_AWARE
behavior.
Posted by vi...@apache.org.
Added test cases for PARTITION_AWARE behavior.
Review: https://reviews.apache.org/r/50706/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/9ba79dae
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/9ba79dae
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/9ba79dae
Branch: refs/heads/master
Commit: 9ba79daead85cb84759582e1aacf41eea0d43054
Parents: 937c85f
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:47:09 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:47:09 2016 -0700
----------------------------------------------------------------------
src/tests/partition_tests.cpp | 514 ++++++++++++++++++++++++++++++++++++-
1 file changed, 510 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/9ba79dae/src/tests/partition_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/partition_tests.cpp b/src/tests/partition_tests.cpp
index 34622b3..efdb49e 100644
--- a/src/tests/partition_tests.cpp
+++ b/src/tests/partition_tests.cpp
@@ -179,7 +179,7 @@ TEST_P(PartitionTest, ReregisterSlavePartitionAware)
// Allow the master to PING the slave, but drop all PONG messages
// from the slave. Note that we don't match on the master / slave
- // PIDs because it's actually the SlaveObserver Process that sends
+ // PIDs because it's actually the `SlaveObserver` process that sends
// the pings.
Future<Message> ping = FUTURE_MESSAGE(
Eq(PingSlaveMessage().GetTypeName()), _, _);
@@ -326,7 +326,7 @@ TEST_P(PartitionTest, ReregisterSlaveNotPartitionAware)
// Allow the master to PING the slave, but drop all PONG messages
// from the slave. Note that we don't match on the master / slave
- // PIDs because it's actually the SlaveObserver Process that sends
+ // PIDs because it's actually the `SlaveObserver` process that sends
// the pings.
Future<Message> ping = FUTURE_MESSAGE(
Eq(PingSlaveMessage().GetTypeName()), _, _);
@@ -458,6 +458,512 @@ TEST_P(PartitionTest, ReregisterSlaveNotPartitionAware)
}
+// This tests that an agent can reregister with the master after a
+// partition in which the master has failed over while the agent was
+// partitioned. We use one agent and two schedulers; one scheduler
+// enables the PARTITION_AWARE capability, while the other does
+// not. Both tasks should survive the reregistration of the partitioned
+// agent: we allow the non-partition-aware task to continue running for
+// backward compatibility with the "non-strict" Mesos 1.0 behavior.
+TEST_P(PartitionTest, PartitionedSlaveReregistrationMasterFailover)
+{
+ Clock::pause();
+
+ master::Flags masterFlags = CreateMasterFlags();
+ masterFlags.registry_strict = GetParam();
+
+ Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
+ ASSERT_SOME(master);
+
+ // Allow the master to PING the slave, but drop all PONG messages
+ // from the slave. Note that we don't match on the master / slave
+ // PIDs because it's actually the `SlaveObserver` process that sends
+ // the pings.
+ Future<Message> ping = FUTURE_MESSAGE(
+ Eq(PingSlaveMessage().GetTypeName()), _, _);
+
+ DROP_PROTOBUFS(PongSlaveMessage(), _, _);
+
+ slave::Flags slaveFlags = CreateSlaveFlags();
+ slaveFlags.resources = "cpus:2;mem:1024";
+
+ StandaloneMasterDetector detector(master.get()->pid);
+
+ Try<Owned<cluster::Slave>> slave = StartSlave(&detector, slaveFlags);
+ ASSERT_SOME(slave);
+
+ // Connect the first scheduler (not PARTITION_AWARE).
+ MockScheduler sched1;
+ TestingMesosSchedulerDriver driver1(&sched1, &detector);
+
+ EXPECT_CALL(sched1, registered(&driver1, _, _));
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched1, resourceOffers(&driver1, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver1.start();
+
+ AWAIT_READY(offers);
+ ASSERT_FALSE(offers.get().empty());
+
+ Offer offer = offers.get()[0];
+
+ Resources taskResources = Resources::parse("cpus:1;mem:512").get();
+
+ EXPECT_TRUE(Resources(offer.resources()).contains(taskResources));
+
+ // Launch `task1` using `sched1`.
+ TaskInfo task1 = createTask(offer.slave_id(), taskResources, "sleep 60");
+
+ Future<TaskStatus> runningStatus1;
+ EXPECT_CALL(sched1, statusUpdate(&driver1, _))
+ .WillOnce(FutureArg<1>(&runningStatus1));
+
+ Future<Nothing> statusUpdateAck1 = FUTURE_DISPATCH(
+ slave.get()->pid, &Slave::_statusUpdateAcknowledgement);
+
+ driver1.launchTasks(offer.id(), {task1});
+
+ AWAIT_READY(runningStatus1);
+ EXPECT_EQ(TASK_RUNNING, runningStatus1.get().state());
+ EXPECT_EQ(task1.task_id(), runningStatus1.get().task_id());
+
+ const SlaveID slaveId = runningStatus1.get().slave_id();
+
+ AWAIT_READY(statusUpdateAck1);
+
+ // Connect the second scheduler (PARTITION_AWARE).
+ FrameworkInfo frameworkInfo2 = DEFAULT_FRAMEWORK_INFO;
+ frameworkInfo2.add_capabilities()->set_type(
+ FrameworkInfo::Capability::PARTITION_AWARE);
+
+ MockScheduler sched2;
+ TestingMesosSchedulerDriver driver2(&sched2, &detector, frameworkInfo2);
+
+ EXPECT_CALL(sched2, registered(&driver2, _, _));
+
+ EXPECT_CALL(sched2, resourceOffers(&driver2, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver2.start();
+
+ AWAIT_READY(offers);
+ ASSERT_FALSE(offers.get().empty());
+
+ offer = offers.get()[0];
+
+ EXPECT_TRUE(Resources(offer.resources()).contains(taskResources));
+
+ // Launch the second task.
+ TaskInfo task2 = createTask(offer.slave_id(), taskResources, "sleep 60");
+
+ Future<TaskStatus> runningStatus2;
+ EXPECT_CALL(sched2, statusUpdate(&driver2, _))
+ .WillOnce(FutureArg<1>(&runningStatus2));
+
+ Future<Nothing> statusUpdateAck2 = FUTURE_DISPATCH(
+ slave.get()->pid, &Slave::_statusUpdateAcknowledgement);
+
+ driver2.launchTasks(offer.id(), {task2});
+
+ AWAIT_READY(runningStatus2);
+ EXPECT_EQ(TASK_RUNNING, runningStatus2.get().state());
+ EXPECT_EQ(task2.task_id(), runningStatus2.get().task_id());
+ EXPECT_EQ(slaveId, runningStatus2.get().slave_id());
+
+ AWAIT_READY(statusUpdateAck2);
+
+ // Now, induce a partition of the slave by having the master
+ // timeout the slave.
+ Future<TaskStatus> lostStatus;
+ EXPECT_CALL(sched1, statusUpdate(&driver1, _))
+ .WillOnce(FutureArg<1>(&lostStatus));
+
+ Future<TaskStatus> unreachableStatus;
+ EXPECT_CALL(sched2, statusUpdate(&driver2, _))
+ .WillOnce(FutureArg<1>(&unreachableStatus));
+
+ // Note that we expect to get `slaveLost` callbacks in both
+ // schedulers, regardless of PARTITION_AWARE.
+ Future<Nothing> slaveLost1;
+ EXPECT_CALL(sched1, slaveLost(&driver1, _))
+ .WillOnce(FutureSatisfy(&slaveLost1));
+
+ Future<Nothing> slaveLost2;
+ EXPECT_CALL(sched2, slaveLost(&driver2, _))
+ .WillOnce(FutureSatisfy(&slaveLost2));
+
+ size_t pings = 0;
+ while (true) {
+ AWAIT_READY(ping);
+ pings++;
+ if (pings == masterFlags.max_agent_ping_timeouts) {
+ break;
+ }
+ ping = FUTURE_MESSAGE(Eq(PingSlaveMessage().GetTypeName()), _, _);
+ Clock::advance(masterFlags.agent_ping_timeout);
+ }
+
+ Clock::advance(masterFlags.agent_ping_timeout);
+
+ // `sched1` should see TASK_LOST.
+ AWAIT_READY(lostStatus);
+ EXPECT_EQ(TASK_LOST, lostStatus.get().state());
+ EXPECT_EQ(TaskStatus::REASON_SLAVE_REMOVED, lostStatus.get().reason());
+ EXPECT_EQ(task1.task_id(), lostStatus.get().task_id());
+ EXPECT_EQ(slaveId, lostStatus.get().slave_id());
+
+ // `sched2` should see TASK_UNREACHABLE.
+ AWAIT_READY(unreachableStatus);
+ EXPECT_EQ(TASK_UNREACHABLE, unreachableStatus.get().state());
+ EXPECT_EQ(TaskStatus::REASON_SLAVE_REMOVED, unreachableStatus.get().reason());
+ EXPECT_EQ(task2.task_id(), unreachableStatus.get().task_id());
+ EXPECT_EQ(slaveId, unreachableStatus.get().slave_id());
+
+ // The master should notify both schedulers that the slave was lost.
+ AWAIT_READY(slaveLost1);
+ AWAIT_READY(slaveLost2);
+
+ EXPECT_CALL(sched1, disconnected(&driver1));
+ EXPECT_CALL(sched2, disconnected(&driver2));
+
+ // Simulate master failover.
+ master->reset();
+ master = StartMaster();
+ ASSERT_SOME(master);
+
+ // Settle the clock to ensure the master finishes recovering the registry.
+ Clock::settle();
+
+ Future<SlaveReregisteredMessage> slaveReregistered = FUTURE_PROTOBUF(
+ SlaveReregisteredMessage(), master.get()->pid, slave.get()->pid);
+
+ Future<Nothing> registered1;
+ EXPECT_CALL(sched1, registered(&driver1, _, _))
+ .WillOnce(FutureSatisfy(®istered1));
+
+ Future<Nothing> registered2;
+ EXPECT_CALL(sched2, registered(&driver2, _, _))
+ .WillOnce(FutureSatisfy(®istered2));
+
+ // Simulate a new master detected event to the slave and the schedulers.
+ detector.appoint(master.get()->pid);
+
+ // Wait for slave to reregister.
+ AWAIT_READY(slaveReregistered);
+
+ // Wait for both schedulers to reregister.
+ AWAIT_READY(registered1);
+ AWAIT_READY(registered2);
+
+ // Have each scheduler perform explicit reconciliation. Both `task1` and
+ // `task2` should be running: `task2` because it is PARTITION_AWARE,
+ // `task1` because the master has failed over and we emulate the old
+ // "non-strict" semantics.
+ TaskStatus status1;
+ status1.mutable_task_id()->CopyFrom(task1.task_id());
+ status1.mutable_slave_id()->CopyFrom(slaveId);
+ status1.set_state(TASK_STAGING); // Dummy value.
+
+ Future<TaskStatus> reconcileUpdate1;
+ EXPECT_CALL(sched1, statusUpdate(&driver1, _))
+ .WillOnce(FutureArg<1>(&reconcileUpdate1));
+
+ driver1.reconcileTasks({status1});
+
+ AWAIT_READY(reconcileUpdate1);
+ EXPECT_EQ(TASK_RUNNING, reconcileUpdate1.get().state());
+ EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate1.get().reason());
+
+ TaskStatus status2;
+ status2.mutable_task_id()->CopyFrom(task2.task_id());
+ status2.mutable_slave_id()->CopyFrom(slaveId);
+ status2.set_state(TASK_STAGING); // Dummy value.
+
+ Future<TaskStatus> reconcileUpdate2;
+ EXPECT_CALL(sched2, statusUpdate(&driver2, _))
+ .WillOnce(FutureArg<1>(&reconcileUpdate2));
+
+ driver2.reconcileTasks({status2});
+
+ AWAIT_READY(reconcileUpdate2);
+ EXPECT_EQ(TASK_RUNNING, reconcileUpdate2.get().state());
+ EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate2.get().reason());
+
+ Clock::resume();
+
+ driver1.stop();
+ driver1.join();
+
+ driver2.stop();
+ driver2.join();
+}
+
+
+// This test case causes a slave to be partitioned while it is running
+// a task for a PARTITION_AWARE scheduler. The scheduler disconnects
+// before the partition heals. Right now, the task is left running as
+// an orphan; once MESOS-4659 is fixed, the task should be shutdown.
+TEST_P(PartitionTest, PartitionedSlaveOrphanedTask)
+{
+ Clock::pause();
+
+ master::Flags masterFlags = CreateMasterFlags();
+ masterFlags.registry_strict = GetParam();
+
+ Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
+ ASSERT_SOME(master);
+
+ // Allow the master to PING the slave, but drop all PONG messages
+ // from the slave. Note that we don't match on the master / slave
+ // PIDs because it's actually the `SlaveObserver` process that sends
+ // the pings.
+ Future<Message> ping = FUTURE_MESSAGE(
+ Eq(PingSlaveMessage().GetTypeName()), _, _);
+
+ DROP_PROTOBUFS(PongSlaveMessage(), _, _);
+
+ StandaloneMasterDetector detector(master.get()->pid);
+
+ Try<Owned<cluster::Slave>> slave = StartSlave(&detector);
+ ASSERT_SOME(slave);
+
+ FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
+ frameworkInfo.add_capabilities()->set_type(
+ FrameworkInfo::Capability::PARTITION_AWARE);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, frameworkInfo, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ Future<FrameworkID> frameworkId;
+ EXPECT_CALL(sched, registered(&driver, _, _))
+ .WillOnce(FutureArg<1>(&frameworkId));
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers));
+
+ driver.start();
+
+ AWAIT_READY(frameworkId);
+ AWAIT_READY(offers);
+ ASSERT_FALSE(offers.get().empty());
+
+ Offer offer = offers.get()[0];
+
+ // Launch `task` using `sched`.
+ TaskInfo task = createTask(offer, "sleep 60");
+
+ Future<TaskStatus> runningStatus;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&runningStatus));
+
+ Future<Nothing> statusUpdateAck = FUTURE_DISPATCH(
+ slave.get()->pid, &Slave::_statusUpdateAcknowledgement);
+
+ driver.launchTasks(offer.id(), {task});
+
+ AWAIT_READY(runningStatus);
+ EXPECT_EQ(TASK_RUNNING, runningStatus.get().state());
+ EXPECT_EQ(task.task_id(), runningStatus.get().task_id());
+
+ const SlaveID slaveId = runningStatus.get().slave_id();
+
+ AWAIT_READY(statusUpdateAck);
+
+ // Now, induce a partition of the slave by having the master
+ // timeout the slave.
+ Future<TaskStatus> unreachableStatus;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&unreachableStatus));
+
+ Future<Nothing> slaveLost;
+ EXPECT_CALL(sched, slaveLost(&driver, _))
+ .WillOnce(FutureSatisfy(&slaveLost));
+
+ size_t pings = 0;
+ while (true) {
+ AWAIT_READY(ping);
+ pings++;
+ if (pings == masterFlags.max_agent_ping_timeouts) {
+ break;
+ }
+ ping = FUTURE_MESSAGE(Eq(PingSlaveMessage().GetTypeName()), _, _);
+ Clock::advance(masterFlags.agent_ping_timeout);
+ }
+
+ Clock::advance(masterFlags.agent_ping_timeout);
+
+ AWAIT_READY(unreachableStatus);
+ EXPECT_EQ(TASK_UNREACHABLE, unreachableStatus.get().state());
+ EXPECT_EQ(TaskStatus::REASON_SLAVE_REMOVED, unreachableStatus.get().reason());
+ EXPECT_EQ(task.task_id(), unreachableStatus.get().task_id());
+ EXPECT_EQ(slaveId, unreachableStatus.get().slave_id());
+
+ AWAIT_READY(slaveLost);
+
+ // Disconnect the scheduler. The default `failover_timeout` is 0, so
+ // the framework's tasks should be shutdown when the slave
+ // reregisters, but this is currently not implemented (MESOS-4659).
+ driver.stop();
+ driver.join();
+
+ // Simulate a master loss event at the slave and then cause the
+ // slave to reregister with the master.
+ detector.appoint(None());
+
+ Future<SlaveReregisteredMessage> slaveReregistered = FUTURE_PROTOBUF(
+ SlaveReregisteredMessage(), master.get()->pid, slave.get()->pid);
+
+ detector.appoint(master.get()->pid);
+
+ AWAIT_READY(slaveReregistered);
+
+ // Check if `task` is still running by querying master's state endpoint.
+ Future<Response> response = process::http::get(
+ master.get()->pid,
+ "state",
+ None(),
+ createBasicAuthHeaders(DEFAULT_CREDENTIAL));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+ AWAIT_EXPECT_RESPONSE_HEADER_EQ(APPLICATION_JSON, "Content-Type", response);
+
+ Try<JSON::Object> parse = JSON::parse<JSON::Object>(response.get().body);
+ ASSERT_SOME(parse);
+
+ JSON::Object state = parse.get();
+ JSON::Array completedFrameworks =
+ state.values["completed_frameworks"].as<JSON::Array>();
+
+ EXPECT_EQ(1u, completedFrameworks.values.size());
+
+ JSON::Object jsonFramework =
+ completedFrameworks.values.front().as<JSON::Object>();
+
+ JSON::String jsonFrameworkId = jsonFramework.values["id"].as<JSON::String>();
+
+ EXPECT_EQ(frameworkId.get(), jsonFrameworkId.value);
+
+ // TODO(neilc): Update this when MESOS-4659 is fixed.
+ JSON::Array orphanTasks = state.values["orphan_tasks"].as<JSON::Array>();
+
+ EXPECT_EQ(1u, orphanTasks.values.size());
+
+ JSON::Object jsonTask = orphanTasks.values.front().as<JSON::Object>();
+ JSON::String jsonTaskId = jsonTask.values["id"].as<JSON::String>();
+
+ EXPECT_EQ(task.task_id(), jsonTaskId.value);
+
+ Clock::resume();
+}
+
+
+// This test checks that when a registered slave reregisters with the
+// master (e.g., because of a spurious Zk leader flag at the slave),
+// the master does not kill any tasks on the slave, even if those
+// tasks are not PARTITION_AWARE.
+TEST_P(PartitionTest, SpuriousSlaveReregistration)
+{
+ Clock::pause();
+
+ master::Flags masterFlags = CreateMasterFlags();
+ masterFlags.registry_strict = GetParam();
+
+ Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
+ ASSERT_SOME(master);
+
+ StandaloneMasterDetector detector(master.get()->pid);
+
+ Try<Owned<cluster::Slave>> slave = StartSlave(&detector);
+ ASSERT_SOME(slave);
+
+ // The framework should not be PARTITION_AWARE, since tasks started
+ // by PARTITION_AWARE frameworks will never be killed on reregistration.
+ FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
+ CHECK(!protobuf::frameworkHasCapability(
+ frameworkInfo, FrameworkInfo::Capability::PARTITION_AWARE));
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, frameworkInfo, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ Future<FrameworkID> frameworkId;
+ EXPECT_CALL(sched, registered(&driver, _, _))
+ .WillOnce(FutureArg<1>(&frameworkId));
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers));
+
+ driver.start();
+
+ AWAIT_READY(frameworkId);
+ AWAIT_READY(offers);
+ ASSERT_FALSE(offers.get().empty());
+
+ Offer offer = offers.get()[0];
+
+ // Launch `task` using `sched`.
+ TaskInfo task = createTask(offer, "sleep 60");
+
+ Future<TaskStatus> runningStatus;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&runningStatus));
+
+ Future<Nothing> statusUpdateAck = FUTURE_DISPATCH(
+ slave.get()->pid, &Slave::_statusUpdateAcknowledgement);
+
+ driver.launchTasks(offer.id(), {task});
+
+ AWAIT_READY(runningStatus);
+ EXPECT_EQ(TASK_RUNNING, runningStatus.get().state());
+ EXPECT_EQ(task.task_id(), runningStatus.get().task_id());
+
+ const SlaveID slaveId = runningStatus.get().slave_id();
+
+ AWAIT_READY(statusUpdateAck);
+
+ // Simulate a master loss event at the slave and then cause the
+ // slave to reregister with the master. From the master's
+ // perspective, the slave reregisters while it was still both
+ // connected and registered.
+ detector.appoint(None());
+
+ Future<SlaveReregisteredMessage> slaveReregistered = FUTURE_PROTOBUF(
+ SlaveReregisteredMessage(), master.get()->pid, slave.get()->pid);
+
+ detector.appoint(master.get()->pid);
+
+ AWAIT_READY(slaveReregistered);
+
+ // Perform explicit reconciliation. The task should still be running.
+ TaskStatus status;
+ status.mutable_task_id()->CopyFrom(task.task_id());
+ status.mutable_slave_id()->CopyFrom(slaveId);
+ status.set_state(TASK_STAGING); // Dummy value.
+
+ Future<TaskStatus> reconcileUpdate;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&reconcileUpdate));
+
+ driver.reconcileTasks({status});
+
+ AWAIT_READY(reconcileUpdate);
+ EXPECT_EQ(TASK_RUNNING, reconcileUpdate.get().state());
+ EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate.get().reason());
+
+ driver.stop();
+ driver.join();
+
+ Clock::resume();
+}
+
+
// This test checks how Mesos behaves when a slave is removed because
// it fails health checks, and then the slave sends a status update
// (because it does not realize that it is partitioned from the
@@ -476,7 +982,7 @@ TEST_P(PartitionTest, PartitionedSlaveStatusUpdates)
// Drop both PINGs from master to slave and PONGs from slave to
// master. Note that we don't match on the master / slave PIDs
- // because it's actually the SlaveObserver Process that sends pings
+ // because it's actually the `SlaveObserver` process that sends pings
// and receives pongs.
DROP_PROTOBUFS(PingSlaveMessage(), _, _);
DROP_PROTOBUFS(PongSlaveMessage(), _, _);
@@ -627,7 +1133,7 @@ TEST_P(PartitionTest, PartitionedSlaveExitedExecutor)
// Allow the master to PING the slave, but drop all PONG messages
// from the slave. Note that we don't match on the master / slave
- // PIDs because it's actually the SlaveObserver Process that sends
+ // PIDs because it's actually the `SlaveObserver` process that sends
// the pings.
Future<Message> ping = FUTURE_MESSAGE(
Eq(PingSlaveMessage().GetTypeName()), _, _);
[17/27] mesos git commit: Updated master to send TASK_UNREACHABLE
task state.
Posted by vi...@apache.org.
Updated master to send TASK_UNREACHABLE task state.
When a task launched by a partition-aware framework is running on an
agent that becomes partitioned from the master, the framework will now
receive TASK_UNREACHABLE, not TASK_LOST.
Similarly, when a partition-aware framework does explicit reconciliation
for an agent ID that appears in the "unreachable" list in the registry,
the master will now return TASK_UNREACHABLE rather than TASK_LOST.
Review: https://reviews.apache.org/r/51805/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/fac32e4d
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/fac32e4d
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/fac32e4d
Branch: refs/heads/master
Commit: fac32e4d7f430d612b9ffc985e4c55a0ae3312fc
Parents: 81fe779
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:48:42 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:48:42 2016 -0700
----------------------------------------------------------------------
src/master/master.cpp | 37 +++++++++++++-------
src/tests/partition_tests.cpp | 62 +++++++++++++++++++--------------
src/tests/reconciliation_tests.cpp | 11 ++++--
3 files changed, 69 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/fac32e4d/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 38ca425..beea5ff 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -5858,9 +5858,9 @@ void Master::markUnreachable(const SlaveID& slaveId)
TimeInfo unreachableTime = protobuf::getCurrentTime();
// Update the registry to move this slave from the list of admitted
- // slaves to the list of unreachable slaves. After this is
- // completed, we can update the master's in-memory state to remove
- // the slave and send TASK_LOST status updates to the frameworks.
+ // slaves to the list of unreachable slaves. After this is complete,
+ // we can remove the slave from the master's in-memory state and
+ // send TASK_UNREACHABLE / TASK_LOST updates to the frameworks.
registrar->apply(Owned<Operation>(
new MarkSlaveUnreachable(slave->info, unreachableTime)))
.onAny(defer(self(),
@@ -5909,19 +5909,25 @@ void Master::_markUnreachable(
// the slave is already removed.
allocator->removeSlave(slave->id);
- // Transition the tasks to TASK_LOST and remove them.
- // TODO(neilc): Update this to send TASK_UNREACHABLE for
- // PARTITION_AWARE frameworks.
+ // Transition tasks to TASK_UNREACHABLE / TASK_LOST and remove them.
+ // We only use TASK_UNREACHABLE if the framework has opted in to the
+ // PARTITION_AWARE capability.
foreachkey (const FrameworkID& frameworkId, utils::copy(slave->tasks)) {
Framework* framework = getFramework(frameworkId);
CHECK_NOTNULL(framework);
+ TaskState newTaskState = TASK_UNREACHABLE;
+ if (!protobuf::frameworkHasCapability(
+ framework->info, FrameworkInfo::Capability::PARTITION_AWARE)) {
+ newTaskState = TASK_LOST;
+ }
+
foreachvalue (Task* task, utils::copy(slave->tasks[frameworkId])) {
const StatusUpdate& update = protobuf::createStatusUpdate(
task->framework_id(),
task->slave_id(),
task->task_id(),
- TASK_LOST,
+ newTaskState,
TaskStatus::SOURCE_MASTER,
None(),
"Slave " + slave->info.hostname() + " is unreachable",
@@ -6120,7 +6126,7 @@ void Master::_reconcileTasks(
// (2) Task is known: send the latest state.
// (3) Task is unknown, slave is registered: TASK_LOST.
// (4) Task is unknown, slave is transitioning: no-op.
- // (5) Task is unknown, slave is unreachable: TASK_LOST.
+ // (5) Task is unknown, slave is unreachable: TASK_UNREACHABLE.
// (6) Task is unknown, slave is unknown: TASK_LOST.
//
// When using a non-strict registry, case (6) may result in
@@ -6190,16 +6196,23 @@ void Master::_reconcileTasks(
<< " for framework " << *framework
<< " because there are transitional agents";
} else if (slaveId.isSome() && slaves.unreachable.contains(slaveId.get())) {
- // (5) Slave is unreachable: TASK_LOST. The status update
- // includes the time at which the slave was marked as
- // unreachable.
+ // (5) Slave is unreachable: TASK_UNREACHABLE. If the framework
+ // does not have the PARTITION_AWARE capability, send TASK_LOST
+ // for backward compatibility. In either case, the status update
+ // also includes the time when the slave was marked unreachable.
TimeInfo unreachableTime = slaves.unreachable[slaveId.get()];
+ TaskState taskState = TASK_UNREACHABLE;
+ if (!protobuf::frameworkHasCapability(
+ framework->info, FrameworkInfo::Capability::PARTITION_AWARE)) {
+ taskState = TASK_LOST;
+ }
+
update = protobuf::createStatusUpdate(
framework->id(),
slaveId.get(),
status.task_id(),
- TASK_LOST,
+ taskState,
TaskStatus::SOURCE_MASTER,
None(),
"Reconciliation: Task is unreachable",
http://git-wip-us.apache.org/repos/asf/mesos/blob/fac32e4d/src/tests/partition_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/partition_tests.cpp b/src/tests/partition_tests.cpp
index d2fce57..c860d2a 100644
--- a/src/tests/partition_tests.cpp
+++ b/src/tests/partition_tests.cpp
@@ -262,9 +262,8 @@ TEST_P(PartitionTest, ReregisterSlavePartitionAware)
Clock::advance(masterFlags.agent_ping_timeout);
- // TODO(neilc): Update this when TASK_UNREACHABLE is introduced.
AWAIT_READY(unreachableStatus);
- EXPECT_EQ(TASK_LOST, unreachableStatus.get().state());
+ EXPECT_EQ(TASK_UNREACHABLE, unreachableStatus.get().state());
EXPECT_EQ(TaskStatus::REASON_SLAVE_REMOVED, unreachableStatus.get().reason());
EXPECT_EQ(task.task_id(), unreachableStatus.get().task_id());
EXPECT_EQ(slaveId, unreachableStatus.get().slave_id());
@@ -272,8 +271,8 @@ TEST_P(PartitionTest, ReregisterSlavePartitionAware)
AWAIT_READY(slaveLost);
JSON::Object stats = Metrics();
- EXPECT_EQ(1, stats.values["master/tasks_lost"]);
- EXPECT_EQ(0, stats.values["master/tasks_unreachable"]);
+ EXPECT_EQ(0, stats.values["master/tasks_lost"]);
+ EXPECT_EQ(1, stats.values["master/tasks_unreachable"]);
EXPECT_EQ(1, stats.values["master/slave_unreachable_scheduled"]);
EXPECT_EQ(1, stats.values["master/slave_unreachable_completed"]);
EXPECT_EQ(1, stats.values["master/slave_removals"]);
@@ -642,10 +641,9 @@ TEST_P(PartitionTest, PartitionedSlaveReregistrationMasterFailover)
EXPECT_EQ(slaveId, lostStatus.get().slave_id());
EXPECT_EQ(partitionTime, lostStatus.get().unreachable_time());
- // `sched2` should see TASK_LOST.
- // TODO(neilc): Update this to expect TASK_UNREACHABLE.
+ // `sched2` should see TASK_UNREACHABLE.
AWAIT_READY(unreachableStatus);
- EXPECT_EQ(TASK_LOST, unreachableStatus.get().state());
+ EXPECT_EQ(TASK_UNREACHABLE, unreachableStatus.get().state());
EXPECT_EQ(TaskStatus::REASON_SLAVE_REMOVED, unreachableStatus.get().reason());
EXPECT_EQ(task2.task_id(), unreachableStatus.get().task_id());
EXPECT_EQ(slaveId, unreachableStatus.get().slave_id());
@@ -805,9 +803,9 @@ TEST_P(PartitionTest, PartitionedSlaveOrphanedTask)
// Now, induce a partition of the slave by having the master
// timeout the slave.
- Future<TaskStatus> lostStatus;
+ Future<TaskStatus> unreachableStatus;
EXPECT_CALL(sched, statusUpdate(&driver, _))
- .WillOnce(FutureArg<1>(&lostStatus));
+ .WillOnce(FutureArg<1>(&unreachableStatus));
Future<Nothing> slaveLost;
EXPECT_CALL(sched, slaveLost(&driver, _))
@@ -835,13 +833,12 @@ TEST_P(PartitionTest, PartitionedSlaveOrphanedTask)
Clock::advance(Milliseconds(100));
- // TODO(neilc): Update this to expect `TASK_UNREACHABLE`.
- AWAIT_READY(lostStatus);
- EXPECT_EQ(TASK_LOST, lostStatus.get().state());
- EXPECT_EQ(TaskStatus::REASON_SLAVE_REMOVED, lostStatus.get().reason());
- EXPECT_EQ(task.task_id(), lostStatus.get().task_id());
- EXPECT_EQ(slaveId, lostStatus.get().slave_id());
- EXPECT_EQ(partitionTime, lostStatus.get().unreachable_time());
+ AWAIT_READY(unreachableStatus);
+ EXPECT_EQ(TASK_UNREACHABLE, unreachableStatus.get().state());
+ EXPECT_EQ(TaskStatus::REASON_SLAVE_REMOVED, unreachableStatus.get().reason());
+ EXPECT_EQ(task.task_id(), unreachableStatus.get().task_id());
+ EXPECT_EQ(slaveId, unreachableStatus.get().slave_id());
+ EXPECT_EQ(partitionTime, unreachableStatus.get().unreachable_time());
AWAIT_READY(slaveLost);
@@ -1325,9 +1322,13 @@ TEST_P(PartitionTest, RegistryGcByCount)
AWAIT_READY(slaveRegisteredMessage1);
const SlaveID slaveId1 = slaveRegisteredMessage1.get().slave_id();
+ FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
+ frameworkInfo.add_capabilities()->set_type(
+ FrameworkInfo::Capability::PARTITION_AWARE);
+
MockScheduler sched;
MesosSchedulerDriver driver(
- &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+ &sched, frameworkInfo, master.get()->pid, DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _));
@@ -1443,7 +1444,7 @@ TEST_P(PartitionTest, RegistryGcByCount)
driver.reconcileTasks({status1});
AWAIT_READY(reconcileUpdate1);
- EXPECT_EQ(TASK_LOST, reconcileUpdate1.get().state());
+ EXPECT_EQ(TASK_UNREACHABLE, reconcileUpdate1.get().state());
EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate1.get().reason());
EXPECT_EQ(partitionTime1, reconcileUpdate1.get().unreachable_time());
@@ -1485,7 +1486,7 @@ TEST_P(PartitionTest, RegistryGcByCount)
driver.reconcileTasks({status3});
AWAIT_READY(reconcileUpdate3);
- EXPECT_EQ(TASK_LOST, reconcileUpdate3.get().state());
+ EXPECT_EQ(TASK_UNREACHABLE, reconcileUpdate3.get().state());
EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate3.get().reason());
EXPECT_EQ(partitionTime2, reconcileUpdate3.get().unreachable_time());
@@ -1558,10 +1559,15 @@ TEST_P(PartitionTest, RegistryGcByCountManySlaves)
Clock::advance(masterFlags.registry_gc_interval);
Clock::settle();
- // Start a scheduler: we verify GC behavior by doing reconciliation.
+ // Start a partition-aware scheduler. We verify GC behavior by doing
+ // reconciliation.
+ FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
+ frameworkInfo.add_capabilities()->set_type(
+ FrameworkInfo::Capability::PARTITION_AWARE);
+
MockScheduler sched;
MesosSchedulerDriver driver(
- &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+ &sched, frameworkInfo, master.get()->pid, DEFAULT_CREDENTIAL);
Future<Nothing> registered;
EXPECT_CALL(sched, registered(&driver, _, _))
@@ -1589,7 +1595,7 @@ TEST_P(PartitionTest, RegistryGcByCountManySlaves)
driver.reconcileTasks({status1});
AWAIT_READY(reconcileUpdate1);
- EXPECT_EQ(TASK_LOST, reconcileUpdate1.get().state());
+ EXPECT_EQ(TASK_UNREACHABLE, reconcileUpdate1.get().state());
EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate1.get().reason());
EXPECT_EQ(unreachableTime, reconcileUpdate1.get().unreachable_time());
@@ -1660,9 +1666,13 @@ TEST_P(PartitionTest, RegistryGcByAge)
AWAIT_READY(slaveRegisteredMessage1);
const SlaveID slaveId1 = slaveRegisteredMessage1.get().slave_id();
+ FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
+ frameworkInfo.add_capabilities()->set_type(
+ FrameworkInfo::Capability::PARTITION_AWARE);
+
MockScheduler sched;
MesosSchedulerDriver driver(
- &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+ &sched, frameworkInfo, master.get()->pid, DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _));
@@ -1782,7 +1792,7 @@ TEST_P(PartitionTest, RegistryGcByAge)
driver.reconcileTasks({status1});
AWAIT_READY(reconcileUpdate1);
- EXPECT_EQ(TASK_LOST, reconcileUpdate1.get().state());
+ EXPECT_EQ(TASK_UNREACHABLE, reconcileUpdate1.get().state());
EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate1.get().reason());
EXPECT_EQ(partitionTime1, reconcileUpdate1.get().unreachable_time());
@@ -1798,7 +1808,7 @@ TEST_P(PartitionTest, RegistryGcByAge)
driver.reconcileTasks({status2});
AWAIT_READY(reconcileUpdate2);
- EXPECT_EQ(TASK_LOST, reconcileUpdate2.get().state());
+ EXPECT_EQ(TASK_UNREACHABLE, reconcileUpdate2.get().state());
EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate2.get().reason());
EXPECT_EQ(partitionTime2, reconcileUpdate2.get().unreachable_time());
@@ -1837,7 +1847,7 @@ TEST_P(PartitionTest, RegistryGcByAge)
driver.reconcileTasks({status4});
AWAIT_READY(reconcileUpdate4);
- EXPECT_EQ(TASK_LOST, reconcileUpdate4.get().state());
+ EXPECT_EQ(TASK_UNREACHABLE, reconcileUpdate4.get().state());
EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate4.get().reason());
EXPECT_EQ(partitionTime2, reconcileUpdate4.get().unreachable_time());
http://git-wip-us.apache.org/repos/asf/mesos/blob/fac32e4d/src/tests/reconciliation_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/reconciliation_tests.cpp b/src/tests/reconciliation_tests.cpp
index 9828ab9..1412090 100644
--- a/src/tests/reconciliation_tests.cpp
+++ b/src/tests/reconciliation_tests.cpp
@@ -230,15 +230,20 @@ TEST_F(ReconciliationTest, TaskStateMatch)
// This test verifies that reconciliation of a task that belongs to an
-// unknown slave results in TASK_LOST.
+// unknown slave results in TASK_LOST, even if the framework has
+// enabled the PARTITION_AWARE capability.
TEST_F(ReconciliationTest, UnknownSlave)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
+ FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
+ frameworkInfo.add_capabilities()->set_type(
+ FrameworkInfo::Capability::PARTITION_AWARE);
+
MockScheduler sched;
MesosSchedulerDriver driver(
- &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+ &sched, frameworkInfo, master.get()->pid, DEFAULT_CREDENTIAL);
Future<FrameworkID> frameworkId;
EXPECT_CALL(sched, registered(&driver, _, _))
@@ -253,7 +258,7 @@ TEST_F(ReconciliationTest, UnknownSlave)
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&update));
- // Create a task status with a random slave id (and task id).
+ // Create a task status with a random slave id and task id.
TaskStatus status;
status.mutable_task_id()->set_value(UUID::random().toString());
status.mutable_slave_id()->set_value(UUID::random().toString());
[05/27] mesos git commit: Added `unreachable_time` to TaskStatus.
Posted by vi...@apache.org.
Added `unreachable_time` to TaskStatus.
This field contains the time at which the master marked an agent as
unreachable. It will only be set for status updates (including
reconciliation updates) that describe tasks running on unreachable
(e.g., partitioned) agents.
The intent is to help frameworks decide how to handle tasks running on
partitioned agents: since the framework might have missed the initial
TASK_LOST/TASK_UNREACHABLE status update (e.g., due to framework
downtime/failover), it might otherwise be difficult for frameworks to
determine how long an agent has been partitioned.
This unreachable timestamp is stored in the registry and loaded as part
of master recovery.
Review: https://reviews.apache.org/r/50845/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/ac269193
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/ac269193
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/ac269193
Branch: refs/heads/master
Commit: ac26919329cb137e09b1a6c8ef500dea2aa908eb
Parents: 06506fa
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:47:26 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:47:26 2016 -0700
----------------------------------------------------------------------
include/mesos/mesos.proto | 6 ++
include/mesos/v1/mesos.proto | 6 ++
src/common/protobuf_utils.cpp | 7 +-
src/common/protobuf_utils.hpp | 3 +-
src/master/master.cpp | 53 ++++++++--
src/master/master.hpp | 11 ++-
src/tests/partition_tests.cpp | 32 ++++++
src/tests/reconciliation_tests.cpp | 168 ++++++++++++++++++++++++++++++++
src/tests/registrar_tests.cpp | 19 +++-
9 files changed, 289 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/ac269193/include/mesos/mesos.proto
----------------------------------------------------------------------
diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto
index 7241c11..2209ea2 100644
--- a/include/mesos/mesos.proto
+++ b/include/mesos/mesos.proto
@@ -1636,6 +1636,12 @@ message TaskStatus {
// Container related information that is resolved dynamically such as
// network address.
optional ContainerStatus container_status = 13;
+
+ // The time (according to the master's clock) when the agent where
+ // this task was running became unreachable. This is only set on
+ // status updates for tasks running on agents that are unreachable
+ // (e.g., partitioned away from the master).
+ optional TimeInfo unreachable_time = 14;
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/ac269193/include/mesos/v1/mesos.proto
----------------------------------------------------------------------
diff --git a/include/mesos/v1/mesos.proto b/include/mesos/v1/mesos.proto
index 04eaeb3..00c6234 100644
--- a/include/mesos/v1/mesos.proto
+++ b/include/mesos/v1/mesos.proto
@@ -1635,6 +1635,12 @@ message TaskStatus {
// Container related information that is resolved dynamically such as
// network address.
optional ContainerStatus container_status = 13;
+
+ // The time (according to the master's clock) when the agent where
+ // this task was running became unreachable. This is only set on
+ // status updates for tasks running on agents that are unreachable
+ // (e.g., partitioned away from the master).
+ optional TimeInfo unreachable_time = 14;
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/ac269193/src/common/protobuf_utils.cpp
----------------------------------------------------------------------
diff --git a/src/common/protobuf_utils.cpp b/src/common/protobuf_utils.cpp
index ed3ac7f..5db4be4 100644
--- a/src/common/protobuf_utils.cpp
+++ b/src/common/protobuf_utils.cpp
@@ -102,7 +102,8 @@ StatusUpdate createStatusUpdate(
const Option<ExecutorID>& executorId,
const Option<bool>& healthy,
const Option<Labels>& labels,
- const Option<ContainerStatus>& containerStatus)
+ const Option<ContainerStatus>& containerStatus,
+ const Option<TimeInfo> unreachableTime)
{
StatusUpdate update;
@@ -150,6 +151,10 @@ StatusUpdate createStatusUpdate(
status->mutable_container_status()->CopyFrom(containerStatus.get());
}
+ if (unreachableTime.isSome()) {
+ status->mutable_unreachable_time()->CopyFrom(unreachableTime.get());
+ }
+
return update;
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/ac269193/src/common/protobuf_utils.hpp
----------------------------------------------------------------------
diff --git a/src/common/protobuf_utils.hpp b/src/common/protobuf_utils.hpp
index 3324838..c5e5a9a 100644
--- a/src/common/protobuf_utils.hpp
+++ b/src/common/protobuf_utils.hpp
@@ -74,7 +74,8 @@ StatusUpdate createStatusUpdate(
const Option<ExecutorID>& executorId = None(),
const Option<bool>& healthy = None(),
const Option<Labels>& labels = None(),
- const Option<ContainerStatus>& containerStatus = None());
+ const Option<ContainerStatus>& containerStatus = None(),
+ const Option<TimeInfo> unreachableTime = None());
StatusUpdate createStatusUpdate(
http://git-wip-us.apache.org/repos/asf/mesos/blob/ac269193/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 89fde24..d5a194b 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -1569,6 +1569,11 @@ Future<Nothing> Master::_recover(const Registry& registry)
slaves.recovered.insert(slave.info().id());
}
+ foreach (const Registry::UnreachableSlave& unreachable,
+ registry.unreachable().slaves()) {
+ slaves.unreachable[unreachable.id()] = unreachable.timestamp();
+ }
+
// Set up a timeout for slaves to re-register.
slaves.recoveredTimer =
delay(flags.agent_reregister_timeout,
@@ -1759,7 +1764,8 @@ Nothing Master::markUnreachableAfterFailover(const Registry::Slave& slave)
if (flags.registry_strict) {
slaves.markingUnreachable.insert(slave.info().id());
- registrar->apply(Owned<Operation>(new MarkSlaveUnreachable(slave.info())))
+ registrar->apply(Owned<Operation>(
+ new MarkSlaveUnreachable(slave.info(), protobuf::getCurrentTime())))
.onAny(defer(self(),
&Self::_markUnreachable,
slave.info(),
@@ -1774,7 +1780,8 @@ Nothing Master::markUnreachableAfterFailover(const Registry::Slave& slave)
const string& message =
"Failed to mark agent " + stringify(slave.info().id()) + " unreachable";
- registrar->apply(Owned<Operation>(new MarkSlaveUnreachable(slave.info())))
+ registrar->apply(Owned<Operation>(
+ new MarkSlaveUnreachable(slave.info(), protobuf::getCurrentTime())))
.onFailed(lambda::bind(fail, message, lambda::_1));
}
@@ -5676,6 +5683,10 @@ void Master::markUnreachable(const SlaveID& slaveId)
// the slave is already removed.
allocator->removeSlave(slave->id);
+ // Use the same timestamp for all status updates sent below; we also
+ // use this timestamp when updating the registry.
+ TimeInfo unreachableTime = protobuf::getCurrentTime();
+
// Transition the tasks to TASK_LOST and remove them, BUT do not
// send updates yet. Rather, build up the updates so that we can can
// send them after the slave has been moved to the unreachable list
@@ -5698,7 +5709,11 @@ void Master::markUnreachable(const SlaveID& slaveId)
"Slave " + slave->info.hostname() + " is unreachable",
TaskStatus::REASON_SLAVE_REMOVED,
(task->has_executor_id() ?
- Option<ExecutorID>(task->executor_id()) : None()));
+ Option<ExecutorID>(task->executor_id()) : None()),
+ None(),
+ None(),
+ None(),
+ unreachableTime);
updateTask(task, update);
removeTask(task);
@@ -5737,6 +5752,7 @@ void Master::markUnreachable(const SlaveID& slaveId)
// Mark the slave as being unreachable.
slaves.registered.remove(slave);
slaves.removed.put(slave->id, Nothing());
+ slaves.unreachable[slave->id] = unreachableTime;
slaves.markingUnreachable.insert(slave->id);
authenticated.erase(slave->pid);
@@ -5755,7 +5771,8 @@ void Master::markUnreachable(const SlaveID& slaveId)
// Update the registry to move this slave from the list of admitted
// slaves to the list of unreachable slaves. Once this is completed,
// we can forward the TASK_LOST updates to the frameworks.
- registrar->apply(Owned<Operation>(new MarkSlaveUnreachable(slave->info)))
+ registrar->apply(Owned<Operation>(
+ new MarkSlaveUnreachable(slave->info, unreachableTime)))
.onAny(defer(self(),
&Self::_markUnreachable,
slave->info,
@@ -5955,9 +5972,10 @@ void Master::_reconcileTasks(
// (2) Task is known: send the latest state.
// (3) Task is unknown, slave is registered: TASK_LOST.
// (4) Task is unknown, slave is transitioning: no-op.
- // (5) Task is unknown, slave is unknown: TASK_LOST.
+ // (5) Task is unknown, slave is unreachable: TASK_LOST.
+ // (6) Task is unknown, slave is unknown: TASK_LOST.
//
- // When using a non-strict registry, case (5) may result in
+ // When using a non-strict registry, case (6) may result in
// a TASK_LOST for a task that may later be non-terminal. This
// is better than no reply at all because the framework can take
// action for TASK_LOST. Later, if the task is running, the
@@ -6023,8 +6041,28 @@ void Master::_reconcileTasks(
LOG(INFO) << "Dropping reconciliation of task " << status.task_id()
<< " for framework " << *framework
<< " because there are transitional agents";
+ } else if (slaveId.isSome() && slaves.unreachable.contains(slaveId.get())) {
+ // (5) Slave is unreachable: TASK_LOST. The status update
+ // includes the time at which the slave was marked as
+ // unreachable.
+ TimeInfo unreachableTime = slaves.unreachable[slaveId.get()];
+
+ update = protobuf::createStatusUpdate(
+ framework->id(),
+ slaveId.get(),
+ status.task_id(),
+ TASK_LOST,
+ TaskStatus::SOURCE_MASTER,
+ None(),
+ "Reconciliation: Task is unreachable",
+ TaskStatus::REASON_RECONCILIATION,
+ None(),
+ None(),
+ None(),
+ None(),
+ unreachableTime);
} else {
- // (5) Task is unknown, slave is unknown: TASK_LOST.
+ // (6) Task is unknown, slave is unknown: TASK_LOST.
update = protobuf::createStatusUpdate(
framework->id(),
slaveId,
@@ -7000,6 +7038,7 @@ void Master::addSlave(
CHECK_NOTNULL(slave);
slaves.removed.erase(slave->id);
+ slaves.unreachable.erase(slave->id);
slaves.registered.put(slave);
link(slave->pid);
http://git-wip-us.apache.org/repos/asf/mesos/blob/ac269193/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index b092de4..6aeb1b9 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -1697,6 +1697,11 @@ private:
// TODO(bmahler): Ideally we could use a cache with set semantics.
Cache<SlaveID, Nothing> removed;
+ // Slaves that have been marked unreachable. We recover this from the
+ // registry, so it includes slaves marked as unreachable by other
+ // instances of the master.
+ hashmap<SlaveID, TimeInfo> unreachable;
+
// This rate limiter is used to limit the removal of slaves failing
// health checks.
// NOTE: Using a 'shared_ptr' here is OK because 'RateLimiter' is
@@ -1942,7 +1947,8 @@ private:
class MarkSlaveUnreachable : public Operation
{
public:
- explicit MarkSlaveUnreachable(const SlaveInfo& _info) : info(_info) {
+ MarkSlaveUnreachable(const SlaveInfo& _info, TimeInfo _unreachableTime)
+ : info(_info), unreachableTime(_unreachableTime) {
CHECK(info.has_id()) << "SlaveInfo is missing the 'id' field";
}
@@ -1970,7 +1976,7 @@ protected:
registry->mutable_unreachable()->add_slaves();
unreachable->mutable_id()->CopyFrom(info.id());
- unreachable->mutable_timestamp()->CopyFrom(protobuf::getCurrentTime());
+ unreachable->mutable_timestamp()->CopyFrom(unreachableTime);
return true; // Mutation.
}
@@ -1982,6 +1988,7 @@ protected:
private:
const SlaveInfo info;
+ const TimeInfo unreachableTime;
};
http://git-wip-us.apache.org/repos/asf/mesos/blob/ac269193/src/tests/partition_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/partition_tests.cpp b/src/tests/partition_tests.cpp
index efdb49e..f962f84 100644
--- a/src/tests/partition_tests.cpp
+++ b/src/tests/partition_tests.cpp
@@ -402,6 +402,15 @@ TEST_P(PartitionTest, ReregisterSlaveNotPartitionAware)
}
Clock::advance(masterFlags.agent_ping_timeout);
+ Clock::settle();
+
+ // Record the time at which we expect the master to have marked the
+ // agent as unhealthy. We then advance the clock -- this shouldn't
+ // do anything, but it ensures that the `unreachable_time` we check
+ // below is computed at the right time.
+ TimeInfo partitionTime = protobuf::getCurrentTime();
+
+ Clock::advance(Milliseconds(100));
// The scheduler should see TASK_LOST because it is not
// PARTITION_AWARE.
@@ -410,6 +419,7 @@ TEST_P(PartitionTest, ReregisterSlaveNotPartitionAware)
EXPECT_EQ(TaskStatus::REASON_SLAVE_REMOVED, lostStatus.get().reason());
EXPECT_EQ(task.task_id(), lostStatus.get().task_id());
EXPECT_EQ(slaveId, lostStatus.get().slave_id());
+ EXPECT_EQ(partitionTime, lostStatus.get().unreachable_time());
AWAIT_READY(slaveLost);
@@ -450,6 +460,7 @@ TEST_P(PartitionTest, ReregisterSlaveNotPartitionAware)
AWAIT_READY(reconcileUpdate);
EXPECT_EQ(TASK_LOST, reconcileUpdate.get().state());
EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate.get().reason());
+ EXPECT_FALSE(reconcileUpdate.get().has_unreachable_time());
Clock::resume();
@@ -608,6 +619,15 @@ TEST_P(PartitionTest, PartitionedSlaveReregistrationMasterFailover)
}
Clock::advance(masterFlags.agent_ping_timeout);
+ Clock::settle();
+
+ // Record the time at which we expect the master to have marked the
+ // agent as unhealthy. We then advance the clock -- this shouldn't
+ // do anything, but it ensures that the `unreachable_time` we check
+ // below is computed at the right time.
+ TimeInfo partitionTime = protobuf::getCurrentTime();
+
+ Clock::advance(Milliseconds(100));
// `sched1` should see TASK_LOST.
AWAIT_READY(lostStatus);
@@ -615,6 +635,7 @@ TEST_P(PartitionTest, PartitionedSlaveReregistrationMasterFailover)
EXPECT_EQ(TaskStatus::REASON_SLAVE_REMOVED, lostStatus.get().reason());
EXPECT_EQ(task1.task_id(), lostStatus.get().task_id());
EXPECT_EQ(slaveId, lostStatus.get().slave_id());
+ EXPECT_EQ(partitionTime, lostStatus.get().unreachable_time());
// `sched2` should see TASK_UNREACHABLE.
AWAIT_READY(unreachableStatus);
@@ -622,6 +643,7 @@ TEST_P(PartitionTest, PartitionedSlaveReregistrationMasterFailover)
EXPECT_EQ(TaskStatus::REASON_SLAVE_REMOVED, unreachableStatus.get().reason());
EXPECT_EQ(task2.task_id(), unreachableStatus.get().task_id());
EXPECT_EQ(slaveId, unreachableStatus.get().slave_id());
+ EXPECT_EQ(partitionTime, unreachableStatus.get().unreachable_time());
// The master should notify both schedulers that the slave was lost.
AWAIT_READY(slaveLost1);
@@ -797,12 +819,22 @@ TEST_P(PartitionTest, PartitionedSlaveOrphanedTask)
}
Clock::advance(masterFlags.agent_ping_timeout);
+ Clock::settle();
+
+ // Record the time at which we expect the master to have marked the
+ // agent as unhealthy. We then advance the clock -- this shouldn't
+ // do anything, but it ensures that the `unreachable_time` we check
+ // below is computed at the right time.
+ TimeInfo partitionTime = protobuf::getCurrentTime();
+
+ Clock::advance(Milliseconds(100));
AWAIT_READY(unreachableStatus);
EXPECT_EQ(TASK_UNREACHABLE, unreachableStatus.get().state());
EXPECT_EQ(TaskStatus::REASON_SLAVE_REMOVED, unreachableStatus.get().reason());
EXPECT_EQ(task.task_id(), unreachableStatus.get().task_id());
EXPECT_EQ(slaveId, unreachableStatus.get().slave_id());
+ EXPECT_EQ(partitionTime, unreachableStatus.get().unreachable_time());
AWAIT_READY(slaveLost);
http://git-wip-us.apache.org/repos/asf/mesos/blob/ac269193/src/tests/reconciliation_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/reconciliation_tests.cpp b/src/tests/reconciliation_tests.cpp
index 8e438bf..a043356 100644
--- a/src/tests/reconciliation_tests.cpp
+++ b/src/tests/reconciliation_tests.cpp
@@ -55,6 +55,7 @@ using mesos::master::detector::StandaloneMasterDetector;
using process::Clock;
using process::Future;
+using process::Message;
using process::Owned;
using process::PID;
using process::Promise;
@@ -65,6 +66,7 @@ using testing::_;
using testing::An;
using testing::AtMost;
using testing::DoAll;
+using testing::Eq;
using testing::Return;
using testing::SaveArg;
@@ -262,6 +264,7 @@ TEST_F(ReconciliationTest, UnknownSlave)
AWAIT_READY(update);
EXPECT_EQ(TASK_LOST, update.get().state());
EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, update.get().reason());
+ EXPECT_FALSE(update.get().has_unreachable_time());
driver.stop();
driver.join();
@@ -318,6 +321,7 @@ TEST_F(ReconciliationTest, UnknownTask)
AWAIT_READY(update);
EXPECT_EQ(TASK_LOST, update.get().state());
EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, update.get().reason());
+ EXPECT_FALSE(update.get().has_unreachable_time());
driver.stop();
driver.join();
@@ -359,6 +363,7 @@ TEST_F(ReconciliationTest, UnknownKillTask)
AWAIT_READY(update);
EXPECT_EQ(TASK_LOST, update.get().state());
EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, update.get().reason());
+ EXPECT_FALSE(update.get().has_unreachable_time());
driver.stop();
driver.join();
@@ -878,6 +883,169 @@ TEST_F(ReconciliationTest, ReconcileStatusUpdateTaskState)
driver.join();
}
+
+TEST_F(ReconciliationTest, PartitionedAgentThenMasterFailover)
+{
+ master::Flags masterFlags = CreateMasterFlags();
+
+ Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
+ ASSERT_SOME(master);
+
+ // Allow the master to PING the slave, but drop all PONG messages
+ // from the slave. Note that we don't match on the master / slave
+ // PIDs because it's actually the `SlaveObserver` process that sends
+ // the pings.
+ Future<Message> ping = FUTURE_MESSAGE(
+ Eq(PingSlaveMessage().GetTypeName()), _, _);
+
+ DROP_PROTOBUFS(PongSlaveMessage(), _, _);
+
+ StandaloneMasterDetector detector(master.get()->pid);
+
+ Try<Owned<cluster::Slave>> slave = StartSlave(&detector);
+ ASSERT_SOME(slave);
+
+ MockScheduler sched;
+ StandaloneMasterDetector schedulerDetector(master.get()->pid);
+ TestingMesosSchedulerDriver driver(&sched, &schedulerDetector);
+
+ EXPECT_CALL(sched, registered(&driver, _, _));
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers));
+
+ driver.start();
+
+ AWAIT_READY(offers);
+ ASSERT_FALSE(offers.get().empty());
+
+ Offer offer = offers.get()[0];
+
+ // Launch `task` using `sched`.
+ TaskInfo task = createTask(offer, "sleep 60");
+
+ Future<TaskStatus> runningStatus;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&runningStatus));
+
+ Future<Nothing> statusUpdateAck = FUTURE_DISPATCH(
+ slave.get()->pid, &Slave::_statusUpdateAcknowledgement);
+
+ driver.launchTasks(offer.id(), {task});
+
+ AWAIT_READY(runningStatus);
+ EXPECT_EQ(TASK_RUNNING, runningStatus.get().state());
+ EXPECT_EQ(task.task_id(), runningStatus.get().task_id());
+
+ const SlaveID slaveId = runningStatus.get().slave_id();
+
+ AWAIT_READY(statusUpdateAck);
+
+ // Now, induce a partition of the slave by having the master
+ // timeout the slave.
+ Future<TaskStatus> lostStatus;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&lostStatus));
+
+ Future<Nothing> slaveLost;
+ EXPECT_CALL(sched, slaveLost(&driver, _))
+ .WillOnce(FutureSatisfy(&slaveLost));
+
+ Clock::pause();
+
+ size_t pings = 0;
+ while (true) {
+ AWAIT_READY(ping);
+ pings++;
+ if (pings == masterFlags.max_agent_ping_timeouts) {
+ break;
+ }
+ ping = FUTURE_MESSAGE(Eq(PingSlaveMessage().GetTypeName()), _, _);
+ Clock::advance(masterFlags.agent_ping_timeout);
+ }
+
+ Clock::advance(masterFlags.agent_ping_timeout);
+ Clock::settle();
+
+ // Record the time at which we expect the master to have marked the
+ // agent as unhealthy. We then advance the clock -- this shouldn't
+ // do anything, but it ensures that the `unreachable_time` we check
+ // below is computed at the right time.
+ TimeInfo partitionTime = protobuf::getCurrentTime();
+
+ Clock::advance(Milliseconds(100));
+
+ AWAIT_READY(lostStatus);
+ EXPECT_EQ(TASK_LOST, lostStatus.get().state());
+ EXPECT_EQ(TaskStatus::REASON_SLAVE_REMOVED, lostStatus.get().reason());
+ EXPECT_EQ(task.task_id(), lostStatus.get().task_id());
+ EXPECT_EQ(slaveId, lostStatus.get().slave_id());
+ EXPECT_EQ(partitionTime, lostStatus.get().unreachable_time());
+
+ AWAIT_READY(slaveLost);
+
+ // Do the first explicit reconciliation before restarting the master.
+ TaskStatus status1;
+ status1.mutable_task_id()->CopyFrom(task.task_id());
+ status1.mutable_slave_id()->CopyFrom(slaveId);
+ status1.set_state(TASK_STAGING); // Dummy value.
+
+ Future<TaskStatus> reconcileUpdate1;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&reconcileUpdate1));
+
+ driver.reconcileTasks({status1});
+
+ AWAIT_READY(reconcileUpdate1);
+ EXPECT_EQ(TASK_LOST, reconcileUpdate1.get().state());
+ EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate1.get().reason());
+ EXPECT_EQ(partitionTime, reconcileUpdate1.get().unreachable_time());
+
+ // Drop all subsequent messages from the slave, since they aren't
+ // useful anymore.
+ DROP_MESSAGES(_, slave.get()->pid, _);
+
+ EXPECT_CALL(sched, disconnected(&driver));
+
+ // Simulate master failover by restarting the master.
+ master->reset();
+ master = StartMaster(masterFlags);
+ ASSERT_SOME(master);
+
+ Future<Nothing> registered;
+ EXPECT_CALL(sched, registered(&driver, _, _))
+ .WillOnce(FutureSatisfy(®istered));
+
+ // Re-register the framework. Note that we don't update the slave's
+ // master detector, so it does not try to reregister.
+ schedulerDetector.appoint(master.get()->pid);
+
+ AWAIT_READY(registered);
+
+ // Do a second explicit reconciliation; we expect to observe the same data.
+ TaskStatus status2;
+ status2.mutable_task_id()->CopyFrom(task.task_id());
+ status2.mutable_slave_id()->CopyFrom(slaveId);
+ status2.set_state(TASK_STAGING); // Dummy value.
+
+ Future<TaskStatus> reconcileUpdate2;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&reconcileUpdate2));
+
+ driver.reconcileTasks({status2});
+
+ AWAIT_READY(reconcileUpdate2);
+ EXPECT_EQ(TASK_LOST, reconcileUpdate2.get().state());
+ EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate2.get().reason());
+ EXPECT_EQ(partitionTime, reconcileUpdate2.get().unreachable_time());
+
+ Clock::resume();
+
+ driver.stop();
+ driver.join();
+}
+
} // namespace tests {
} // namespace internal {
} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/ac269193/src/tests/registrar_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/registrar_tests.cpp b/src/tests/registrar_tests.cpp
index b04fc92..7ea00df 100644
--- a/src/tests/registrar_tests.cpp
+++ b/src/tests/registrar_tests.cpp
@@ -236,7 +236,9 @@ TEST_P(RegistrarTest, Recover)
AWAIT_EXPECT_FAILED(
registrar.apply(Owned<Operation>(new AdmitSlave(slave))));
AWAIT_EXPECT_FAILED(
- registrar.apply(Owned<Operation>(new MarkSlaveUnreachable(slave))));
+ registrar.apply(
+ Owned<Operation>(
+ new MarkSlaveUnreachable(slave, protobuf::getCurrentTime()))));
AWAIT_EXPECT_FAILED(
registrar.apply(Owned<Operation>(new MarkSlaveReachable(slave))));
AWAIT_EXPECT_FAILED(
@@ -249,7 +251,8 @@ TEST_P(RegistrarTest, Recover)
Future<bool> admit = registrar.apply(
Owned<Operation>(new AdmitSlave(slave)));
Future<bool> unreachable = registrar.apply(
- Owned<Operation>(new MarkSlaveUnreachable(slave)));
+ Owned<Operation>(
+ new MarkSlaveUnreachable(slave, protobuf::getCurrentTime())));
Future<bool> reachable = registrar.apply(
Owned<Operation>(new MarkSlaveReachable(slave)));
Future<bool> remove = registrar.apply(
@@ -334,18 +337,24 @@ TEST_P(RegistrarTest, MarkUnreachable)
AWAIT_TRUE(registrar.apply(Owned<Operation>(new AdmitSlave(info1))));
AWAIT_TRUE(
- registrar.apply(Owned<Operation>(new MarkSlaveUnreachable(info1))));
+ registrar.apply(
+ Owned<Operation>(
+ new MarkSlaveUnreachable(info1, protobuf::getCurrentTime()))));
AWAIT_TRUE(
registrar.apply(Owned<Operation>(new MarkSlaveReachable(info1))));
AWAIT_TRUE(
- registrar.apply(Owned<Operation>(new MarkSlaveUnreachable(info1))));
+ registrar.apply(
+ Owned<Operation>(
+ new MarkSlaveUnreachable(info1, protobuf::getCurrentTime()))));
// If a slave is already unreachable, trying to mark it unreachable
// again should fail.
AWAIT_FALSE(
- registrar.apply(Owned<Operation>(new MarkSlaveUnreachable(info1))));
+ registrar.apply(
+ Owned<Operation>(
+ new MarkSlaveUnreachable(info1, protobuf::getCurrentTime()))));
}
[14/27] mesos git commit: Tweaked logging for UnregisterSlaveMessage
handler.
Posted by vi...@apache.org.
Tweaked logging for UnregisterSlaveMessage handler.
Remove a redundant log line: rather than logging when we receive
an UnregisterSlaveMessage, it is sufficient to log how we handle
that message. Also ensure that we log when ignoring an agent
unregister message for an unknown agent, and do some minor code
cleanup.
Review: https://reviews.apache.org/r/51706/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/0d85e7eb
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/0d85e7eb
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/0d85e7eb
Branch: refs/heads/master
Commit: 0d85e7eb4b2da50ebd79f4ef53183745eba42f48
Parents: 9dad0b0
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:48:23 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:48:23 2016 -0700
----------------------------------------------------------------------
src/master/master.cpp | 25 ++++++++++++++-----------
1 file changed, 14 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/0d85e7eb/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 5089b99..4e0df8a 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -5458,20 +5458,23 @@ void Master::unregisterSlave(const UPID& from, const SlaveID& slaveId)
{
++metrics->messages_unregister_slave;
- LOG(INFO) << "Asked to unregister agent " << slaveId;
-
Slave* slave = slaves.registered.get(slaveId);
- if (slave != nullptr) {
- if (slave->pid != from) {
- LOG(WARNING) << "Ignoring unregister agent message from " << from
- << " because it is not the agent " << slave->pid;
- return;
- }
- removeSlave(slave,
- "the agent unregistered",
- metrics->slave_removals_reason_unregistered);
+ if (slave == nullptr) {
+ LOG(WARNING) << "Ignoring unregister agent message from " << from
+ << " for unknown agent";
+ return;
}
+
+ if (slave->pid != from) {
+ LOG(WARNING) << "Ignoring unregister agent message from " << from
+ << " because it is not the agent " << slave->pid;
+ return;
+ }
+
+ removeSlave(slave,
+ "the agent unregistered",
+ metrics->slave_removals_reason_unregistered);
}
[07/27] mesos git commit: Clarified a log message.
Posted by vi...@apache.org.
Clarified a log message.
Review: https://reviews.apache.org/r/51020/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/df037de6
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/df037de6
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/df037de6
Branch: refs/heads/master
Commit: df037de65facd02ac179cecd7dc5c62243694fc5
Parents: 3cecdbd
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:47:41 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:47:41 2016 -0700
----------------------------------------------------------------------
src/master/master.cpp | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/df037de6/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 34de462..008ff32 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -5835,8 +5835,8 @@ void Master::_markUnreachable(
// Notify all frameworks of the lost slave.
foreachvalue (Framework* framework, frameworks.registered) {
LOG(INFO) << "Notifying framework " << *framework << " of lost agent "
- << slaveInfo.id() << " (" << slaveInfo.hostname() << ") "
- << "after recovering";
+ << slaveInfo.id() << " (" << slaveInfo.hostname() << ")";
+
LostSlaveMessage message;
message.mutable_slave_id()->MergeFrom(slaveInfo.id());
framework->send(message);
@@ -7295,8 +7295,8 @@ void Master::_removeSlave(
// Notify all frameworks of the lost slave.
foreachvalue (Framework* framework, frameworks.registered) {
LOG(INFO) << "Notifying framework " << *framework << " of lost agent "
- << slaveInfo.id() << " (" << slaveInfo.hostname() << ") "
- << "after recovering";
+ << slaveInfo.id() << " (" << slaveInfo.hostname() << ")";
+
LostSlaveMessage message;
message.mutable_slave_id()->MergeFrom(slaveInfo.id());
framework->send(message);
[20/27] mesos git commit: Added benchmarks for new registry
operations.
Posted by vi...@apache.org.
Added benchmarks for new registry operations.
These benchmarks cover:
1. The time taken to make all registered agents unreachable and then
reachable. This is similar to what happens during a severe network
partition.
2. The time taken to GC a significant fraction (50%) of the
unreachable list from the registry.
Review: https://reviews.apache.org/r/51909/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/aae81968
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/aae81968
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/aae81968
Branch: refs/heads/master
Commit: aae819684f61f4cb811e0629731a57bdd6f8f3fe
Parents: 47bd3e4
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:49:04 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:49:04 2016 -0700
----------------------------------------------------------------------
src/tests/registrar_tests.cpp | 141 ++++++++++++++++++++++++++++++++++++-
1 file changed, 139 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/aae81968/src/tests/registrar_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/registrar_tests.cpp b/src/tests/registrar_tests.cpp
index 745cded..928bc66 100644
--- a/src/tests/registrar_tests.cpp
+++ b/src/tests/registrar_tests.cpp
@@ -1263,8 +1263,6 @@ TEST_P(Registrar_BENCHMARK_Test, Performance)
Registrar registrar(flags, state);
AWAIT_READY(registrar.recover(master));
- vector<SlaveInfo> infos;
-
Attributes attributes = Attributes::parse("foo:bar;baz:quux");
Resources resources =
Resources::parse("cpus(*):1.0;mem(*):512;disk(*):2048").get();
@@ -1272,6 +1270,7 @@ TEST_P(Registrar_BENCHMARK_Test, Performance)
size_t slaveCount = GetParam();
// Create slaves.
+ vector<SlaveInfo> infos;
for (size_t i = 0; i < slaveCount; ++i) {
// Simulate real slave information.
SlaveInfo info;
@@ -1333,6 +1332,144 @@ TEST_P(Registrar_BENCHMARK_Test, Performance)
cout << "Removed " << slaveCount << " agents in " << watch.elapsed() << endl;
}
+
+// Test the performance of marking all registered slaves unreachable,
+// then marking them reachable again. This might occur if there is a
+// network partition and then the partition heals.
+TEST_P(Registrar_BENCHMARK_Test, MarkUnreachableThenReachable)
+{
+ Registrar registrar(flags, state);
+ AWAIT_READY(registrar.recover(master));
+
+ Attributes attributes = Attributes::parse("foo:bar;baz:quux");
+ Resources resources =
+ Resources::parse("cpus(*):1.0;mem(*):512;disk(*):2048").get();
+
+ size_t slaveCount = GetParam();
+
+ // Create slaves.
+ vector<SlaveInfo> infos;
+ for (size_t i = 0; i < slaveCount; ++i) {
+ // Simulate real slave information.
+ SlaveInfo info;
+ info.set_hostname("localhost");
+ info.mutable_id()->set_value(
+ string("201310101658-2280333834-5050-48574-") + stringify(i));
+ info.mutable_resources()->MergeFrom(resources);
+ info.mutable_attributes()->MergeFrom(attributes);
+ infos.push_back(info);
+ }
+
+ // Admit slaves.
+ Stopwatch watch;
+ watch.start();
+ Future<bool> result;
+ foreach (const SlaveInfo& info, infos) {
+ result = registrar.apply(Owned<Operation>(new AdmitSlave(info)));
+ }
+ AWAIT_READY_FOR(result, Minutes(5));
+ LOG(INFO) << "Admitted " << slaveCount << " agents in " << watch.elapsed();
+
+ // Shuffle the slaves so that we mark them unreachable in random
+ // order (same as in production).
+ std::random_shuffle(infos.begin(), infos.end());
+
+ // Mark all slaves unreachable.
+ TimeInfo unreachableTime = protobuf::getCurrentTime();
+
+ watch.start();
+ foreach (const SlaveInfo& info, infos) {
+ result = registrar.apply(
+ Owned<Operation>(new MarkSlaveUnreachable(info, unreachableTime)));
+ }
+ AWAIT_READY_FOR(result, Minutes(5));
+ cout << "Marked " << slaveCount << " agents unreachable in "
+ << watch.elapsed() << endl;
+
+ // Shuffles the slaves again so that we mark them reachable in
+ // random order (same as in production).
+ std::random_shuffle(infos.begin(), infos.end());
+
+ // Mark all slaves reachable.
+ watch.start();
+ foreach (const SlaveInfo& info, infos) {
+ result = registrar.apply(
+ Owned<Operation>(new MarkSlaveReachable(info)));
+ }
+ AWAIT_READY_FOR(result, Minutes(5));
+ cout << "Marked " << slaveCount << " agents reachable in "
+ << watch.elapsed() << endl;
+}
+
+
+// Test the performance of garbage collecting a large portion of the
+// unreachable list in a single operation. We use a fixed percentage
+// at the moment (50%).
+TEST_P(Registrar_BENCHMARK_Test, GcManyAgents)
+{
+ Registrar registrar(flags, state);
+ AWAIT_READY(registrar.recover(master));
+
+ Attributes attributes = Attributes::parse("foo:bar;baz:quux");
+ Resources resources =
+ Resources::parse("cpus(*):1.0;mem(*):512;disk(*):2048").get();
+
+ size_t slaveCount = GetParam();
+
+ // Create slaves.
+ vector<SlaveInfo> infos;
+ for (size_t i = 0; i < slaveCount; ++i) {
+ // Simulate real slave information.
+ SlaveInfo info;
+ info.set_hostname("localhost");
+ info.mutable_id()->set_value(
+ string("201310101658-2280333834-5050-48574-") + stringify(i));
+ info.mutable_resources()->MergeFrom(resources);
+ info.mutable_attributes()->MergeFrom(attributes);
+ infos.push_back(info);
+ }
+
+ // Admit slaves.
+ Stopwatch watch;
+ watch.start();
+ Future<bool> result;
+ foreach (const SlaveInfo& info, infos) {
+ result = registrar.apply(Owned<Operation>(new AdmitSlave(info)));
+ }
+ AWAIT_READY_FOR(result, Minutes(5));
+ LOG(INFO) << "Admitted " << slaveCount << " agents in " << watch.elapsed();
+
+ // Shuffle the slaves so that we mark them unreachable in random
+ // order (same as in production).
+ std::random_shuffle(infos.begin(), infos.end());
+
+ // Mark all slaves unreachable.
+ TimeInfo unreachableTime = protobuf::getCurrentTime();
+
+ watch.start();
+ foreach (const SlaveInfo& info, infos) {
+ result = registrar.apply(
+ Owned<Operation>(new MarkSlaveUnreachable(info, unreachableTime)));
+ }
+ AWAIT_READY_FOR(result, Minutes(5));
+ LOG(INFO) << "Marked " << slaveCount << " agents unreachable in "
+ << watch.elapsed() << endl;
+
+ // Prepare to GC the first half of the unreachable list.
+ hashset<SlaveID> toRemove;
+ for (size_t i = 0; (i * 2) < slaveCount; i++) {
+ const SlaveInfo& info = infos[i];
+ toRemove.insert(info.id());
+ }
+
+ // Do GC.
+ watch.start();
+ result = registrar.apply(Owned<Operation>(new PruneUnreachable(toRemove)));
+ AWAIT_READY_FOR(result, Minutes(5));
+ cout << "Garbage collected " << toRemove.size() << " agents in "
+ << watch.elapsed() << endl;
+}
+
} // namespace tests {
} // namespace internal {
} // namespace mesos {
[19/27] mesos git commit: Added test case for registry GC race
condition.
Posted by vi...@apache.org.
Added test case for registry GC race condition.
If GC occurs concurrently with another operation that changes the
registry (e.g., reregistration of an agent that is also going to be
removed by the GC operation), the GC might not be able to prune as
many entries as expected from the registry.
Along the way, this commit updates the master's logging to emit
a warning in this situation. It also corrects an inaccuracy: we
should report the number of agents we actually pruned from the
registry, not the number we _attempted_ to reclaim.
Review: https://reviews.apache.org/r/51891/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/47bd3e45
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/47bd3e45
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/47bd3e45
Branch: refs/heads/master
Commit: 47bd3e458b8cfcfd1ca0114419e56069da15453f
Parents: 6172da9
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:48:54 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:48:54 2016 -0700
----------------------------------------------------------------------
src/master/master.cpp | 28 +++-
src/tests/partition_tests.cpp | 280 +++++++++++++++++++++++++++++++++++++
2 files changed, 301 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/47bd3e45/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index beea5ff..e4499b5 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -1766,16 +1766,30 @@ void Master::_doRegistryGc(
// `PruneUnreachable` registry operation should never fail.
CHECK(registrarResult.get());
- // TODO(neilc): Add a metric for # of agents discarded from the registry?
- LOG(INFO) << "Garbage collected " << toRemove.size()
- << " unreachable agents from the registry";
-
- // Update in-memory state to be consistent with registry changes.
+ // Update in-memory state to be consistent with registry changes. If
+ // there was a concurrent registry operation that also modified the
+ // unreachable list (e.g., an agent in `toRemove` concurrently
+ // reregistered), entries in `toRemove` might not appear in
+ // `slaves.unreachable`.
+ //
+ // TODO(neilc): It would be nice to verify that the effect of these
+ // in-memory updates is equivalent to the changes made by the registry
+ // operation, but there isn't an easy way to do that.
+ size_t numRemoved = 0;
foreach (const SlaveID& slave, toRemove) {
- // NOTE: `slave` might not appear in `slaves.unreachable` if there
- // have been concurrent changes.
+ if (!slaves.unreachable.contains(slave)) {
+ LOG(WARNING) << "Failed to garbage collect " << slave
+ << " from the unreachable list";
+ continue;
+ }
+
slaves.unreachable.erase(slave);
+ numRemoved++;
}
+
+ // TODO(neilc): Add a metric for # of agents discarded from the registry?
+ LOG(INFO) << "Garbage collected " << numRemoved
+ << " unreachable agents from the registry";
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/47bd3e45/src/tests/partition_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/partition_tests.cpp b/src/tests/partition_tests.cpp
index c860d2a..895a073 100644
--- a/src/tests/partition_tests.cpp
+++ b/src/tests/partition_tests.cpp
@@ -1881,6 +1881,286 @@ TEST_P(PartitionTest, RegistryGcByAge)
}
+// This test checks what happens when these two operations race:
+// garbage collecting some slave IDs from the "unreachable" list in
+// the registry and moving a slave from the unreachable list back to
+// the admitted list. We add three agents to the unreachable list and
+// configure GC to only keep a single agent. Concurrently with GC
+// running, we arrange for one of those agents to reregister with the
+// master.
+TEST_P(PartitionTest, RegistryGcRace2)
+{
+ master::Flags masterFlags = CreateMasterFlags();
+ masterFlags.registry_strict = GetParam();
+ masterFlags.registry_max_agent_count = 1;
+
+ Clock::pause();
+
+ Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
+ ASSERT_SOME(master);
+
+ // Allow the master to PING the slave, but drop all PONG messages
+ // from the slave. Note that we don't match on the master / slave
+ // PIDs because it's actually the `SlaveObserver` process that sends
+ // the pings.
+ Future<Message> ping = FUTURE_MESSAGE(
+ Eq(PingSlaveMessage().GetTypeName()), _, _);
+
+ DROP_PROTOBUFS(PongSlaveMessage(), _, _);
+
+ Future<SlaveRegisteredMessage> slaveRegisteredMessage1 =
+ FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _);
+
+ Owned<MasterDetector> detector1 = master.get()->createDetector();
+
+ Try<Owned<cluster::Slave>> slave1 = StartSlave(detector1.get());
+ ASSERT_SOME(slave1);
+
+ // Wait for the slave to register and get the slave id.
+ AWAIT_READY(slaveRegisteredMessage1);
+ SlaveID slaveId1 = slaveRegisteredMessage1.get().slave_id();
+
+ FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
+ frameworkInfo.add_capabilities()->set_type(
+ FrameworkInfo::Capability::PARTITION_AWARE);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, frameworkInfo, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(&driver, _, _));
+
+ Future<Nothing> resourceOffers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureSatisfy(&resourceOffers))
+ .WillRepeatedly(Return());
+
+ driver.start();
+
+ AWAIT_READY(resourceOffers);
+
+ // Induce a partition of the slave.
+ size_t pings1 = 0;
+ while (true) {
+ AWAIT_READY(ping);
+ pings1++;
+ if (pings1 == masterFlags.max_agent_ping_timeouts) {
+ break;
+ }
+ ping = FUTURE_MESSAGE(Eq(PingSlaveMessage().GetTypeName()), _, _);
+ Clock::advance(masterFlags.agent_ping_timeout);
+ }
+
+ Future<Nothing> slaveLost1;
+ EXPECT_CALL(sched, slaveLost(&driver, _))
+ .WillOnce(FutureSatisfy(&slaveLost1));
+
+ EXPECT_CALL(sched, offerRescinded(&driver, _))
+ .WillRepeatedly(Return());
+
+ Clock::advance(masterFlags.agent_ping_timeout);
+
+ AWAIT_READY(slaveLost1);
+
+ // `slave1` will try to re-register below when we advance the clock.
+ // Prevent this by dropping all future messages from it.
+ DROP_MESSAGES(_, slave1.get()->pid, _);
+
+ Future<SlaveRegisteredMessage> slaveRegisteredMessage2 =
+ FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _);
+
+ StandaloneMasterDetector detector2(master.get()->pid);
+
+ Try<Owned<cluster::Slave>> slave2 = StartSlave(&detector2);
+ ASSERT_SOME(slave2);
+
+ // Wait for the slave to register and get the slave id.
+ AWAIT_READY(slaveRegisteredMessage2);
+ SlaveID slaveId2 = slaveRegisteredMessage2.get().slave_id();
+
+ // Induce a partition of the slave.
+ size_t pings2 = 0;
+ while (true) {
+ AWAIT_READY(ping);
+ pings2++;
+ if (pings2 == masterFlags.max_agent_ping_timeouts) {
+ break;
+ }
+ ping = FUTURE_MESSAGE(Eq(PingSlaveMessage().GetTypeName()), _, _);
+ Clock::advance(masterFlags.agent_ping_timeout);
+ }
+
+ Future<Nothing> slaveLost2;
+ EXPECT_CALL(sched, slaveLost(&driver, _))
+ .WillOnce(FutureSatisfy(&slaveLost2));
+
+ Clock::advance(masterFlags.agent_ping_timeout);
+
+ AWAIT_READY(slaveLost2);
+
+ // `slave2` will try to re-register below when we advance the clock.
+ // Prevent this by dropping the next `ReregisterSlaveMessage` from it.
+ Future<Message> reregisterSlave2 =
+ DROP_MESSAGE(Eq(ReregisterSlaveMessage().GetTypeName()),
+ slave2.get()->pid,
+ master.get()->pid);
+
+ Future<SlaveRegisteredMessage> slaveRegisteredMessage3 =
+ FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _);
+
+ Owned<MasterDetector> detector3 = master.get()->createDetector();
+
+ Try<Owned<cluster::Slave>> slave3 = StartSlave(detector3.get());
+ ASSERT_SOME(slave3);
+
+ // Wait for the slave to register and get the slave id.
+ AWAIT_READY(slaveRegisteredMessage3);
+ SlaveID slaveId3 = slaveRegisteredMessage3.get().slave_id();
+
+ // Induce a partition of the slave.
+ size_t pings3 = 0;
+ while (true) {
+ AWAIT_READY(ping);
+ pings3++;
+ if (pings3 == masterFlags.max_agent_ping_timeouts) {
+ break;
+ }
+ ping = FUTURE_MESSAGE(Eq(PingSlaveMessage().GetTypeName()), _, _);
+ Clock::advance(masterFlags.agent_ping_timeout);
+ }
+
+ Future<Nothing> slaveLost3;
+ EXPECT_CALL(sched, slaveLost(&driver, _))
+ .WillOnce(FutureSatisfy(&slaveLost3));
+
+ Clock::advance(masterFlags.agent_ping_timeout);
+
+ AWAIT_READY(slaveLost3);
+ AWAIT_READY(reregisterSlave2);
+
+ // `slave3` will try to re-register below when we advance the clock.
+ // Prevent this by dropping all future messages from it.
+ DROP_MESSAGES(_, slave3.get()->pid, _);
+
+ // Cause `slave2` to reregister with the master. We expect the
+ // master to update the registry to mark the slave as reachable; we
+ // intercept the registry operation.
+ Future<Owned<master::Operation>> markReachable;
+ Promise<bool> markReachableContinue;
+ EXPECT_CALL(*master.get()->registrar.get(), apply(_))
+ .WillOnce(DoAll(FutureArg<0>(&markReachable),
+ Return(markReachableContinue.future())));
+
+ detector2.appoint(master.get()->pid);
+
+ AWAIT_READY(markReachable);
+ EXPECT_NE(
+ nullptr,
+ dynamic_cast<master::MarkSlaveReachable*>(
+ markReachable.get().get()));
+
+ // Trigger GC. Because GC has been configured to preserve a single
+ // unreachable slave (the slave marked unreachable most recently),
+ // this should result in attempting to prune `slave1` and `slave2`
+ // from the unreachable list. We intercept the registry operation to
+ // force the race condition with the reregistration of `slave2`.
+ Future<Owned<master::Operation>> pruneUnreachable;
+ Promise<bool> pruneUnreachableContinue;
+ EXPECT_CALL(*master.get()->registrar.get(), apply(_))
+ .WillOnce(DoAll(FutureArg<0>(&pruneUnreachable),
+ Return(pruneUnreachableContinue.future())));
+
+ Clock::advance(masterFlags.registry_gc_interval);
+
+ AWAIT_READY(pruneUnreachable);
+ EXPECT_NE(
+ nullptr,
+ dynamic_cast<master::PruneUnreachable*>(
+ pruneUnreachable.get().get()));
+
+ // Apply the registry operation to mark the slave reachable, then
+ // pass the result back to the master to allow it to continue. We
+ // validate that `slave2` is reregistered successfully.
+ Future<SlaveReregisteredMessage> slaveReregisteredMessage =
+ FUTURE_PROTOBUF(SlaveReregisteredMessage(),
+ master.get()->pid,
+ slave2.get()->pid);
+
+ Future<bool> applyReachable =
+ master.get()->registrar->unmocked_apply(markReachable.get());
+
+ AWAIT_READY(applyReachable);
+ markReachableContinue.set(applyReachable.get());
+
+ AWAIT_READY(slaveReregisteredMessage);
+
+ // Apply the registry operation to prune the unreachable list, then
+ // pass the result back to the master to allow it to continue.
+ Future<bool> applyPrune =
+ master.get()->registrar->unmocked_apply(pruneUnreachable.get());
+
+ AWAIT_READY(applyPrune);
+ pruneUnreachableContinue.set(applyPrune.get());
+
+ // We expect that `slave1` has been removed from the unreachable
+ // list, `slave2` is registered, and `slave3` is still in the
+ // unreachable list. We use reconciliation to verify this.
+
+ TaskStatus status1;
+ status1.mutable_task_id()->set_value(UUID::random().toString());
+ status1.mutable_slave_id()->CopyFrom(slaveId1);
+ status1.set_state(TASK_STAGING); // Dummy value.
+
+ Future<TaskStatus> reconcileUpdate1;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&reconcileUpdate1));
+
+ driver.reconcileTasks({status1});
+
+ AWAIT_READY(reconcileUpdate1);
+ EXPECT_EQ(TASK_LOST, reconcileUpdate1.get().state());
+ EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate1.get().reason());
+ EXPECT_FALSE(reconcileUpdate1.get().has_unreachable_time());
+
+ TaskStatus status2;
+ status2.mutable_task_id()->set_value(UUID::random().toString());
+ status2.mutable_slave_id()->CopyFrom(slaveId2);
+ status2.set_state(TASK_STAGING); // Dummy value.
+
+ Future<TaskStatus> reconcileUpdate2;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&reconcileUpdate2));
+
+ driver.reconcileTasks({status2});
+
+ AWAIT_READY(reconcileUpdate2);
+ EXPECT_EQ(TASK_LOST, reconcileUpdate2.get().state());
+ EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate2.get().reason());
+ EXPECT_FALSE(reconcileUpdate2.get().has_unreachable_time());
+
+ TaskStatus status3;
+ status3.mutable_task_id()->set_value(UUID::random().toString());
+ status3.mutable_slave_id()->CopyFrom(slaveId3);
+ status3.set_state(TASK_STAGING); // Dummy value.
+
+ Future<TaskStatus> reconcileUpdate3;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&reconcileUpdate3));
+
+ driver.reconcileTasks({status3});
+
+ AWAIT_READY(reconcileUpdate3);
+ EXPECT_EQ(TASK_UNREACHABLE, reconcileUpdate3.get().state());
+ EXPECT_EQ(TaskStatus::REASON_RECONCILIATION, reconcileUpdate3.get().reason());
+ EXPECT_TRUE(reconcileUpdate3.get().has_unreachable_time());
+
+ driver.stop();
+ driver.join();
+
+ Clock::resume();
+}
+
+
// This test checks that the master behaves correctly if a slave fails
// health checks twice. At present, this can only occur if the registry
// operation to mark the slave unreachable takes so long that the
[24/27] mesos git commit: Removed dependency on "registry_strict" in
master failover.
Posted by vi...@apache.org.
Removed dependency on "registry_strict" in master failover.
When the master fails over, agents have `agent_reregister_timeout` to
reregister with the new master. Any agents that fail to reregister
within the timeout will be marked unreachable in the registry.
Previously, frameworks would only receive a `slaveLost` callback for
such agents if the master was running in "registry_strict" mode. This
commit changes the master to always inform frameworks about lost agents,
regardless of the "registry_strict" flag.
Review: https://reviews.apache.org/r/51955/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/905204e5
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/905204e5
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/905204e5
Branch: refs/heads/master
Commit: 905204e54748c5842f96997db41f9ff14d9246ab
Parents: ef9211f
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:49:23 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:49:23 2016 -0700
----------------------------------------------------------------------
src/master/master.cpp | 27 ++++----------
src/tests/master_tests.cpp | 81 -----------------------------------------
2 files changed, 7 insertions(+), 101 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/905204e5/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 763c5e7..66a672f 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -1880,27 +1880,14 @@ Nothing Master::markUnreachableAfterFailover(const Registry::Slave& slave)
TimeInfo unreachableTime = protobuf::getCurrentTime();
- if (flags.registry_strict) {
- slaves.markingUnreachable.insert(slave.info().id());
+ slaves.markingUnreachable.insert(slave.info().id());
- registrar->apply(Owned<Operation>(
- new MarkSlaveUnreachable(slave.info(), unreachableTime)))
- .onAny(defer(self(),
- &Self::_markUnreachableAfterFailover,
- slave.info(),
- lambda::_1));
- } else {
- // When a non-strict registry is in use, we want to ensure the
- // registry is used in a write-only manner. Therefore we remove
- // the slave from the registry but we do not inform the
- // framework.
- const string& message =
- "Failed to mark agent " + stringify(slave.info().id()) + " unreachable";
-
- registrar->apply(Owned<Operation>(
- new MarkSlaveUnreachable(slave.info(), unreachableTime)))
- .onFailed(lambda::bind(fail, message, lambda::_1));
- }
+ registrar->apply(Owned<Operation>(
+ new MarkSlaveUnreachable(slave.info(), unreachableTime)))
+ .onAny(defer(self(),
+ &Self::_markUnreachableAfterFailover,
+ slave.info(),
+ lambda::_1));
return Nothing();
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/905204e5/src/tests/master_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/master_tests.cpp b/src/tests/master_tests.cpp
index 6c49ab3..a32ac12 100644
--- a/src/tests/master_tests.cpp
+++ b/src/tests/master_tests.cpp
@@ -1967,87 +1967,6 @@ TEST_F(MasterTest, RecoveredSlaveCanReregister)
}
-// This test ensures that a non-strict registry is write-only by
-// inducing a slave removal during recovery. After which, we expect
-// that the framework is *not* informed, and we expect that the
-// slave can re-register successfully.
-TEST_F(MasterTest, NonStrictRegistryWriteOnly)
-{
- // Step 1: Start a master.
- master::Flags masterFlags = CreateMasterFlags();
- masterFlags.registry_strict = false;
-
- Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
- ASSERT_SOME(master);
-
- // Step 2: Start a slave.
- Future<SlaveRegisteredMessage> slaveRegisteredMessage =
- FUTURE_PROTOBUF(SlaveRegisteredMessage(), master.get()->pid, _);
-
- // Reuse slaveFlags so both StartSlave() use the same work_dir.
- slave::Flags slaveFlags = this->CreateSlaveFlags();
-
- Owned<MasterDetector> detector = master.get()->createDetector();
- Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), slaveFlags);
- ASSERT_SOME(slave);
-
- AWAIT_READY(slaveRegisteredMessage);
-
- // Step 3: Stop the slave while the master is down.
- master->reset();
- slave.get()->terminate();
- slave->reset();
-
- // Step 4: Restart the master.
- master = StartMaster(masterFlags);
- ASSERT_SOME(master);
-
- // Step 5: Start a scheduler.
- MockScheduler sched;
- MesosSchedulerDriver driver(
- &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
-
- Future<Nothing> registered;
- EXPECT_CALL(sched, registered(&driver, _, _))
- .WillOnce(FutureSatisfy(®istered));
-
- EXPECT_CALL(sched, resourceOffers(&driver, _))
- .WillRepeatedly(Return()); // Ignore offers.
-
- driver.start();
-
- AWAIT_READY(registered);
-
- // Step 6: Advance the clock and make sure the slave is not
- // removed!
- Future<Nothing> slaveLost;
- EXPECT_CALL(sched, slaveLost(&driver, _))
- .WillRepeatedly(FutureSatisfy(&slaveLost));
-
- Clock::pause();
- Clock::advance(masterFlags.agent_reregister_timeout);
- Clock::settle();
-
- ASSERT_TRUE(slaveLost.isPending());
-
- Clock::resume();
-
- // Step 7: Now expect the slave to be able to re-register,
- // according to the non-strict semantics.
- Future<SlaveReregisteredMessage> slaveReregisteredMessage =
- FUTURE_PROTOBUF(SlaveReregisteredMessage(), master.get()->pid, _);
-
- detector = master.get()->createDetector();
- slave = StartSlave(detector.get(), slaveFlags);
- ASSERT_SOME(slave);
-
- AWAIT_READY(slaveReregisteredMessage);
-
- driver.stop();
- driver.join();
-}
-
-
// This test ensures that slave removals during master recovery
// are rate limited.
TEST_F(MasterTest, RateLimitRecoveredSlaveRemoval)
[16/27] mesos git commit: Fixed races between "unreachable" and
"unregister" slave transitions.
Posted by vi...@apache.org.
Fixed races between "unreachable" and "unregister" slave transitions.
Now that we update the registry before updating the master's state when
performing these transitions, it is possible for the master to already
be marking a slave unreachable when an `UnregisterSlaveMessage` is
received, or vice versa. Detect these situations and ignore whichever
transition is triggered second.
Review: https://reviews.apache.org/r/51707/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/81fe779b
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/81fe779b
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/81fe779b
Branch: refs/heads/master
Commit: 81fe779b46e48eff2fde51e8d21402cb7eee006b
Parents: 5e66002
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Sep 19 15:48:36 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Sep 19 15:48:36 2016 -0700
----------------------------------------------------------------------
src/master/master.cpp | 24 ++++-
src/tests/slave_tests.cpp | 229 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 252 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/81fe779b/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 3095f8a..38ca425 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -5839,6 +5839,12 @@ void Master::markUnreachable(const SlaveID& slaveId)
return;
}
+ if (slaves.removing.contains(slaveId)) {
+ LOG(WARNING) << "Not marking agent " << slaveId
+ << " unreachable because it is unregistering";
+ return;
+ }
+
LOG(INFO) << "Marking agent " << *slave
<< " unreachable: health check timed out";
@@ -7284,10 +7290,26 @@ void Master::removeSlave(
{
CHECK_NOTNULL(slave);
- LOG(INFO) << "Removing agent " << *slave << ": " << message;
+ // It would be better to remove the slave here instead of continuing
+ // to mark it unreachable, but probably not worth the complexity.
+ if (slaves.markingUnreachable.contains(slave->id)) {
+ LOG(WARNING) << "Ignoring removal of agent " << *slave
+ << " that is in the process of being marked unreachable";
+ return;
+ }
+
+ // This should not be possible, but we protect against it anyway for
+ // the sake of paranoia.
+ if (slaves.removing.contains(slave->id)) {
+ LOG(WARNING) << "Ignoring removal of agent " << *slave
+ << " that is in the process of being removed";
+ return;
+ }
slaves.removing.insert(slave->id);
+ LOG(INFO) << "Removing agent " << *slave << ": " << message;
+
// Remove this slave from the registrar. Note that we update the
// registry BEFORE we update the master's in-memory state; this
// means that until the registry operation has completed, the slave
http://git-wip-us.apache.org/repos/asf/mesos/blob/81fe779b/src/tests/slave_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/slave_tests.cpp b/src/tests/slave_tests.cpp
index aa30118..8ad3547 100644
--- a/src/tests/slave_tests.cpp
+++ b/src/tests/slave_tests.cpp
@@ -2886,6 +2886,235 @@ TEST_F(SlaveTest, HealthCheckUnregisterRace)
}
+// This test checks that the master behaves correctly when a slave
+// fails health checks and is in the process of being marked
+// unreachable in the registry, but concurrently the slave unregisters
+// from the master.
+TEST_F(SlaveTest, UnreachableThenUnregisterRace)
+{
+ master::Flags masterFlags = CreateMasterFlags();
+ Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
+ ASSERT_SOME(master);
+
+ // Set these expectations up before we spawn the slave so that we
+ // don't miss the first PING.
+ Future<Message> ping = FUTURE_MESSAGE(
+ Eq(PingSlaveMessage().GetTypeName()), _, _);
+
+ // Drop all the PONGs to simulate slave partition.
+ DROP_PROTOBUFS(PongSlaveMessage(), _, _);
+
+ Owned<MasterDetector> detector = master.get()->createDetector();
+ Try<Owned<cluster::Slave>> slave = StartSlave(detector.get());
+ ASSERT_SOME(slave);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(&driver, _, _));
+
+ Future<Nothing> resourceOffers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureSatisfy(&resourceOffers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ // Need to make sure the framework AND slave have registered with
+ // master. Waiting for resource offers should accomplish both.
+ AWAIT_READY(resourceOffers);
+
+ Clock::pause();
+
+ EXPECT_CALL(sched, offerRescinded(&driver, _))
+ .Times(AtMost(1));
+
+ Future<Nothing> slaveLost;
+ EXPECT_CALL(sched, slaveLost(&driver, _))
+ .WillOnce(FutureSatisfy(&slaveLost));
+
+ // Now advance through the PINGs.
+ size_t pings = 0;
+ while (true) {
+ AWAIT_READY(ping);
+ pings++;
+ if (pings == masterFlags.max_agent_ping_timeouts) {
+ break;
+ }
+ ping = FUTURE_MESSAGE(Eq(PingSlaveMessage().GetTypeName()), _, _);
+ Clock::advance(masterFlags.agent_ping_timeout);
+ }
+
+ // Intercept the next registry operation. This operation should be
+ // attempting to mark the slave unreachable.
+ Future<Owned<master::Operation>> markUnreachable;
+ Promise<bool> markUnreachableContinue;
+ EXPECT_CALL(*master.get()->registrar.get(), apply(_))
+ .WillOnce(DoAll(FutureArg<0>(&markUnreachable),
+ Return(markUnreachableContinue.future())));
+
+ Clock::advance(masterFlags.agent_ping_timeout);
+
+ AWAIT_READY(markUnreachable);
+ EXPECT_NE(
+ nullptr,
+ dynamic_cast<master::MarkSlaveUnreachable*>(
+ markUnreachable.get().get()));
+
+ // Cause the slave to shutdown gracefully by sending it SIGUSR1.
+ // This should result in the slave sending `UnregisterSlaveMessage`
+ // to the master. Normally, the master would then remove the slave
+ // from the registry, but since the slave is already being marked
+ // unreachable, the master should ignore the unregister message.
+ Future<UnregisterSlaveMessage> unregisterSlaveMessage =
+ FUTURE_PROTOBUF(
+ UnregisterSlaveMessage(),
+ slave.get()->pid,
+ master.get()->pid);
+
+ EXPECT_CALL(*master.get()->registrar.get(), apply(_))
+ .Times(0);
+
+ kill(getpid(), SIGUSR1);
+
+ AWAIT_READY(unregisterSlaveMessage);
+
+ // Apply the registry operation to mark the slave unreachable, then
+ // pass the result back to the master to allow it to continue.
+ Future<bool> applyUnreachable =
+ master.get()->registrar->unmocked_apply(markUnreachable.get());
+
+ AWAIT_READY(applyUnreachable);
+ markUnreachableContinue.set(applyUnreachable.get());
+
+ AWAIT_READY(slaveLost);
+
+ Clock::resume();
+
+ driver.stop();
+ driver.join();
+}
+
+
+// This test checks that the master behaves correctly when a slave is
+// in the process of unregistering from the master when it is marked
+// unreachable.
+TEST_F(SlaveTest, UnregisterThenUnreachableRace)
+{
+ master::Flags masterFlags = CreateMasterFlags();
+ Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
+ ASSERT_SOME(master);
+
+ // Set these expectations up before we spawn the slave so that we
+ // don't miss the first PING.
+ Future<Message> ping = FUTURE_MESSAGE(
+ Eq(PingSlaveMessage().GetTypeName()), _, _);
+
+ // Drop all the PONGs to simulate slave partition.
+ DROP_PROTOBUFS(PongSlaveMessage(), _, _);
+
+ Owned<MasterDetector> detector = master.get()->createDetector();
+ Try<Owned<cluster::Slave>> slave = StartSlave(detector.get());
+ ASSERT_SOME(slave);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(&driver, _, _));
+
+ Future<vector<Offer>> resourceOffers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&resourceOffers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ // Need to make sure the framework AND slave have registered with
+ // master. Waiting for resource offers should accomplish both.
+ AWAIT_READY(resourceOffers);
+
+ ASSERT_EQ(1u, resourceOffers.get().size());
+ SlaveID slaveId = resourceOffers.get()[0].slave_id();
+
+ Clock::pause();
+
+ // Simulate the slave shutting down gracefully. This might happen
+ // normally if the slave receives SIGUSR1. However, we don't use
+ // that approach here, because that would also result in an `exited`
+ // event at the master; we want to test the case where the slave
+ // begins to shutdown but the socket hasn't been closed yet. Hence,
+ // we spoof the `UnregisterSlaveMessage`.
+ //
+ // When the master receives the `UnregisterSlaveMessage`, it should
+ // attempt to remove the slave from the registry.
+ Future<Owned<master::Operation>> removeSlave;
+ Promise<bool> removeSlaveContinue;
+ EXPECT_CALL(*master.get()->registrar.get(), apply(_))
+ .WillOnce(DoAll(FutureArg<0>(&removeSlave),
+ Return(removeSlaveContinue.future())));
+
+ process::dispatch(master.get()->pid,
+ &Master::unregisterSlave,
+ slave.get()->pid,
+ slaveId);
+
+ AWAIT_READY(removeSlave);
+ EXPECT_NE(
+ nullptr,
+ dynamic_cast<master::RemoveSlave*>(removeSlave.get().get()));
+
+ // Next, cause the slave to fail health checks; master will attempt
+ // to mark it unreachable.
+ size_t pings = 0;
+ while (true) {
+ AWAIT_READY(ping);
+ pings++;
+ if (pings == masterFlags.max_agent_ping_timeouts) {
+ break;
+ }
+ ping = FUTURE_MESSAGE(Eq(PingSlaveMessage().GetTypeName()), _, _);
+ Clock::advance(masterFlags.agent_ping_timeout);
+ }
+
+ // We expect the `SlaveObserver` to dispatch a message to the master
+ // to mark the slave unreachable. The master should ignore this
+ // request because the slave is already being removed.
+ Future<Nothing> unreachableDispatch =
+ FUTURE_DISPATCH(master.get()->pid, &Master::markUnreachable);
+
+ EXPECT_CALL(*master.get()->registrar.get(), apply(_))
+ .Times(0);
+
+ Clock::advance(masterFlags.agent_ping_timeout);
+
+ AWAIT_READY(unreachableDispatch);
+
+ EXPECT_CALL(sched, offerRescinded(&driver, _))
+ .Times(AtMost(1));
+
+ Future<Nothing> slaveLost;
+ EXPECT_CALL(sched, slaveLost(&driver, _))
+ .WillOnce(FutureSatisfy(&slaveLost));
+
+ // Apply the registry operation to remove the slave, then pass the
+ // result back to the master to allow it to continue.
+ Future<bool> applyRemove =
+ master.get()->registrar->unmocked_apply(removeSlave.get());
+
+ AWAIT_READY(applyRemove);
+ removeSlaveContinue.set(applyRemove.get());
+
+ AWAIT_READY(slaveLost);
+
+ Clock::resume();
+
+ driver.stop();
+ driver.join();
+}
+
+
// This test ensures that a killTask() can happen between runTask()
// and _run() and then gets "handled properly". This means that
// the task never gets started, but also does not get lost. The end