You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ya...@apache.org on 2017/08/28 23:07:28 UTC
[1/2] mesos git commit: Checkpoint the agent's decision to recover as
a new instance.
Repository: mesos
Updated Branches:
refs/heads/master 0adda9e2f -> 418012815
Checkpoint the agent's decision to recover as a new instance.
After a reboot the agent checkpoints new boot ID after recovering and
agent info after registering with the master. If new agent info is
incompatible with the checkpointed one, the agent will recover as a new
one. But if the agent is restarted before registering it will fail to
start again, because new boot ID will already be checkpointed.
This patch fixes it by checkpointing the agent's decision to recover as
a new one by removing the "latest" meta symlink.
Review: https://reviews.apache.org/r/61661/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/5e4c9d4b
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/5e4c9d4b
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/5e4c9d4b
Branch: refs/heads/master
Commit: 5e4c9d4b354058cdd90a5c61700ef3cdb788c4cf
Parents: 0adda9e
Author: Ilya Pronin <ip...@twopensource.com>
Authored: Mon Aug 28 15:48:48 2017 -0700
Committer: Jiang Yan Xu <xu...@apple.com>
Committed: Mon Aug 28 15:48:48 2017 -0700
----------------------------------------------------------------------
src/slave/slave.cpp | 8 ++++++++
1 file changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/5e4c9d4b/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index 3e880b3..25a1653 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -6098,6 +6098,14 @@ Future<Nothing> Slave::recover(const Try<state::State>& state)
// Cleaning up the slave state to avoid any state recovery for the
// old agent.
slaveState = None();
+
+ // Remove the "latest" symlink if it exists to "checkpoint" the
+ // decision to recover as a new agent.
+ const string& latest = paths::getLatestSlavePath(metaDir);
+ if (os::exists(latest)) {
+ CHECK_SOME(os::rm(latest))
+ << "Failed to remove latest symlink '" << latest << "'";
+ }
} else {
info = slaveState->info.get(); // Recover the slave info.
[2/2] mesos git commit: Added
SlaveRecoveryTest.RebootWithSlaveInfoMismatchAndRestart test.
Posted by ya...@apache.org.
Added SlaveRecoveryTest.RebootWithSlaveInfoMismatchAndRestart test.
The new test verifies that the agent removes the 'latest' symlink when
it decides to recover as a new one after reboot. And that it can
successfully recover again if restarted before checkpointing new agent
info after reregistration.
Review: https://reviews.apache.org/r/61662/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/41801281
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/41801281
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/41801281
Branch: refs/heads/master
Commit: 418012815f669bfb9a6ca08ecef409c7ae33eb28
Parents: 5e4c9d4
Author: Ilya Pronin <ip...@twopensource.com>
Authored: Mon Aug 28 16:04:24 2017 -0700
Committer: Jiang Yan Xu <xu...@apple.com>
Committed: Mon Aug 28 16:07:15 2017 -0700
----------------------------------------------------------------------
src/tests/slave_recovery_tests.cpp | 106 ++++++++++++++++++++++++++++++++
1 file changed, 106 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/41801281/src/tests/slave_recovery_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/slave_recovery_tests.cpp b/src/tests/slave_recovery_tests.cpp
index c2e9e86..0e46748 100644
--- a/src/tests/slave_recovery_tests.cpp
+++ b/src/tests/slave_recovery_tests.cpp
@@ -2780,6 +2780,112 @@ TYPED_TEST(SlaveRecoveryTest, RebootWithSlaveInfoMismatch)
}
+// When the agent is down we modify the BOOT_ID_FILE to simulate a
+// reboot and change the resources flag to cause a mismatch between
+// the recovered agent's info and the one of the new agent to make it
+// recover as a new one. We restart the recovered agent before it
+// registers with the master to ensure it still recovers as a new
+// agent after that.
+TYPED_TEST(SlaveRecoveryTest, RebootWithSlaveInfoMismatchAndRestart)
+{
+ Try<Owned<cluster::Master>> master = this->StartMaster();
+ ASSERT_SOME(master);
+
+ slave::Flags flags = this->CreateSlaveFlags();
+ flags.resources = "cpus:8;mem:4096;disk:2048";
+
+ Fetcher fetcher(flags);
+
+ Try<TypeParam*> _containerizer = TypeParam::create(flags, true, &fetcher);
+ ASSERT_SOME(_containerizer);
+ Owned<slave::Containerizer> containerizer(_containerizer.get());
+
+ Owned<MasterDetector> detector = master.get()->createDetector();
+
+ Try<Owned<cluster::Slave>> slave =
+ this->StartSlave(detector.get(), containerizer.get(), flags);
+ ASSERT_SOME(slave);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(_, _, _));
+
+ // Capture offer in order to get the agent ID.
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(_, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(offers);
+ ASSERT_EQ(1u, offers->size());
+ SlaveID slaveId1 = offers.get()[0].slave_id();
+
+ EXPECT_CALL(sched, offerRescinded(_, _))
+ .Times(AtMost(1));
+
+ slave.get()->terminate();
+
+ // Modify the boot ID to simulate a reboot.
+ ASSERT_SOME(os::write(
+ paths::getBootIdPath(paths::getMetaRootDir(flags.work_dir)),
+ "rebooted! ;)"));
+
+ // Change agent's resources to cause agent info mismatch.
+ flags.resources = "cpus:4;mem:2048;disk:2048";
+
+ _containerizer = TypeParam::create(flags, true, &fetcher);
+ ASSERT_SOME(_containerizer);
+ containerizer.reset(_containerizer.get());
+
+ // Capture offer in order to get the new agent ID.
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ Clock::pause();
+ Clock::settle();
+
+ // Simulate an agent restart before reregistration completes.
+ Future<RegisterSlaveMessage> registerSlaveMessage =
+ DROP_PROTOBUF(RegisterSlaveMessage(), _, master.get()->pid);
+
+ slave = this->StartSlave(detector.get(), containerizer.get(), flags);
+ ASSERT_SOME(slave);
+
+ Clock::advance(flags.registration_backoff_factor);
+ Clock::settle();
+
+ AWAIT_READY(registerSlaveMessage);
+
+ slave.get()->terminate();
+
+ Clock::resume();
+
+ // Verify that the last agent instance removed the "latest" symlink
+ // when it detected agent info discrepancy.
+ const string latest =
+ paths::getLatestSlavePath(paths::getMetaRootDir(flags.work_dir));
+ ASSERT_FALSE(os::exists(latest));
+
+ // Start the agent again to verify that it still successfully
+ // recovers as a new one.
+ slave = this->StartSlave(detector.get(), containerizer.get(), flags);
+ ASSERT_SOME(slave);
+
+ // Verify that the agent registered as a new one.
+ AWAIT_READY(offers);
+ ASSERT_EQ(1u, offers->size());
+ EXPECT_NE(slaveId1, offers->at(0).slave_id());
+
+ driver.stop();
+ driver.join();
+}
+
+
// When the slave is down we remove the "latest" symlink in the
// executor's run directory, to simulate a situation where the
// recovered slave (--no-strict) cannot recover the executor and