You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ga...@apache.org on 2018/09/20 22:04:54 UTC
[mesos] branch master updated (8df68cc -> 961d122)
This is an automated email from the ASF dual-hosted git repository.
gaston pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git.
from 8df68cc Reserve sort output vector in the sorters.
new da69a03 Made authentication tests independent of wall clock time.
new 961d122 Added a test to verify agent authentication retry backoff logic.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/slave/slave.hpp | 3 +-
src/tests/authentication_tests.cpp | 241 +++++++++++++++++++++++++++++++++----
src/tests/mock_slave.cpp | 10 ++
src/tests/mock_slave.hpp | 8 ++
4 files changed, 235 insertions(+), 27 deletions(-)
[mesos] 02/02: Added a test to verify agent authentication retry
backoff logic.
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gaston pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 961d12219893aae2978d194140d3a71b0379d4ac
Author: Meng Zhu <mz...@mesosphere.io>
AuthorDate: Thu Sep 20 14:55:03 2018 -0700
Added a test to verify agent authentication retry backoff logic.
This test verifies that the agent backs-off properly when
retrying authentication according to the configured parameters.
Also mocked `Slave::authenticate()` for this test.
Review: https://reviews.apache.org/r/68354/
---
src/slave/slave.hpp | 3 +-
src/tests/authentication_tests.cpp | 109 +++++++++++++++++++++++++++++++++++++
src/tests/mock_slave.cpp | 10 ++++
src/tests/mock_slave.hpp | 8 +++
4 files changed, 129 insertions(+), 1 deletion(-)
diff --git a/src/slave/slave.hpp b/src/slave/slave.hpp
index 6757eae..0bd3401 100644
--- a/src/slave/slave.hpp
+++ b/src/slave/slave.hpp
@@ -491,7 +491,8 @@ public:
// not receive a ping.
void pingTimeout(process::Future<Option<MasterInfo>> future);
- void authenticate(Duration minTimeout, Duration maxTimeout);
+ // Made virtual for testing purpose.
+ virtual void authenticate(Duration minTimeout, Duration maxTimeout);
// Helper routines to lookup a framework/executor.
Framework* getFramework(const FrameworkID& frameworkId) const;
diff --git a/src/tests/authentication_tests.cpp b/src/tests/authentication_tests.cpp
index bbb7e10..1973810 100644
--- a/src/tests/authentication_tests.cpp
+++ b/src/tests/authentication_tests.cpp
@@ -44,6 +44,7 @@ using mesos::master::detector::StandaloneMasterDetector;
using testing::_;
using testing::Eq;
using testing::Return;
+using testing::SaveArg;
namespace mesos {
namespace internal {
@@ -426,6 +427,114 @@ TEST_F(AuthenticationTest, RetrySlaveAuthentication)
ASSERT_NE("", slaveRegisteredMessage->slave_id().value());
}
+// Verify that the agent backs off properly when retrying authentication
+// according to the configured parameters.
+TEST_F(AuthenticationTest, SlaveAuthenticationRetryBackoff)
+{
+ Clock::pause();
+
+ Try<Owned<cluster::Master>> master = StartMaster();
+ ASSERT_SOME(master);
+
+ Owned<MasterDetector> detector = master.get()->createDetector();
+
+ slave::Flags slaveFlags = CreateSlaveFlags();
+ slaveFlags.authentication_timeout_min = Seconds(5);
+ slaveFlags.authentication_timeout_max = Minutes(1);
+ slaveFlags.authentication_backoff_factor = Seconds(1);
+
+ // Expected retry timeout range:
+ //
+ // [min, min + factor * 2^1]
+ // [min, min + factor * 2^2]
+ // ...
+ // [min, min + factor * 2^N]
+ // ...
+ // [min, max] // Stop at max.
+ Duration expected[7][2] = {
+ {Seconds(5), Seconds(7)},
+ {Seconds(5), Seconds(9)},
+ {Seconds(5), Seconds(13)},
+ {Seconds(5), Seconds(21)},
+ {Seconds(5), Seconds(37)},
+ {Seconds(5), Minutes(1)},
+ {Seconds(5), Minutes(1)}
+ };
+
+ Try<Owned<cluster::Slave>> slave =
+ StartSlave(detector.get(), slaveFlags, true);
+ ASSERT_SOME(slave);
+
+ // Drop the first authentication attempt.
+ Future<Nothing> authenticate;
+ Duration minTimeout, maxTimeout;
+ EXPECT_CALL(*slave.get()->mock(), authenticate(_, _))
+ .WillOnce(DoAll(
+ SaveArg<0>(&minTimeout),
+ SaveArg<1>(&maxTimeout),
+ FutureSatisfy(&authenticate)))
+ .RetiresOnSaturation();
+
+ slave.get()->start();
+
+ // Trigger the first authentication request.
+ Clock::advance(slaveFlags.authentication_backoff_factor);
+ AWAIT_READY(authenticate);
+
+ for (int i = 0; i < 7; i++) {
+ EXPECT_EQ(minTimeout, expected[i][0]);
+ EXPECT_EQ(maxTimeout, expected[i][1]);
+
+ // Drop the authenticate message from the slave to incur retry.
+ Future<AuthenticateMessage> authenticateMessage =
+ DROP_PROTOBUF(AuthenticateMessage(), _, _);
+
+ // Drop the retry call and manually issue the retry call (instead
+ // of invoking the unmocked call directly in the expectation. We do this
+ // so that, even though clock is advanced for `authentication_timeout_max`
+ // in each iteration, we can still guarantee:
+ //
+ // (1) Retry only happens once in each iteration;
+ //
+ // (2) At the end of each iteration, the authentication timeout timer is
+ // not started. The timer will start only when we manually issue the
+ // retry call in the next iteration.
+ EXPECT_CALL(*slave.get()->mock(), authenticate(_, _))
+ .WillOnce(DoAll(
+ SaveArg<0>(&minTimeout),
+ SaveArg<1>(&maxTimeout),
+ FutureSatisfy(&authenticate)))
+ .RetiresOnSaturation();
+
+ slave.get()->mock()->unmocked_authenticate(minTimeout, maxTimeout);
+
+ // Slave should not retry until `slaveFlags.authentication_timeout_min`.
+ Clock::advance(slaveFlags.authentication_timeout_min - Milliseconds(1));
+ Clock::settle();
+ EXPECT_TRUE(authenticate.isPending());
+
+ // Slave will retry at least once in
+ // `slaveFlags.authentication_timeout_max`.
+ Clock::advance(
+ slaveFlags.authentication_timeout_max -
+ slaveFlags.authentication_timeout_min + Milliseconds(1));
+ Clock::settle();
+
+ AWAIT_READY(authenticateMessage);
+ AWAIT_READY(authenticate);
+ }
+
+ Future<AuthenticationCompletedMessage> authenticationCompletedMessage =
+ FUTURE_PROTOBUF(AuthenticationCompletedMessage(), _, _);
+
+ slave.get()->mock()->unmocked_authenticate(minTimeout, maxTimeout);
+
+ Clock::advance(slaveFlags.authentication_timeout_max);
+
+ // Slave should be able to get authenticated.
+ AWAIT_READY(authenticationCompletedMessage);
+}
+
// This test ensures that when the master sees a new authentication
// request for a particular agent or scheduler (we just test the
diff --git a/src/tests/mock_slave.cpp b/src/tests/mock_slave.cpp
index 94a5b0d..a78ca9c 100644
--- a/src/tests/mock_slave.cpp
+++ b/src/tests/mock_slave.cpp
@@ -134,6 +134,8 @@ MockSlave::MockSlave(
.WillRepeatedly(Invoke(this, &MockSlave::unmocked_runTaskGroup));
EXPECT_CALL(*this, killTask(_, _))
.WillRepeatedly(Invoke(this, &MockSlave::unmocked_killTask));
+ EXPECT_CALL(*this, authenticate(_, _))
+ .WillRepeatedly(Invoke(this, &MockSlave::unmocked_authenticate));
EXPECT_CALL(*this, removeFramework(_))
.WillRepeatedly(Invoke(this, &MockSlave::unmocked_removeFramework));
EXPECT_CALL(*this, __recover(_))
@@ -251,6 +253,14 @@ void MockSlave::unmocked_killTask(
}
+void MockSlave::unmocked_authenticate(
+ Duration minTimeout,
+ Duration maxTimeout)
+{
+ slave::Slave::authenticate(minTimeout, maxTimeout);
+}
+
+
void MockSlave::unmocked_removeFramework(slave::Framework* framework)
{
slave::Slave::removeFramework(framework);
diff --git a/src/tests/mock_slave.hpp b/src/tests/mock_slave.hpp
index 9a74bf3..3c0d602 100644
--- a/src/tests/mock_slave.hpp
+++ b/src/tests/mock_slave.hpp
@@ -191,6 +191,14 @@ public:
const process::UPID& from,
const KillTaskMessage& killTaskMessage);
+ MOCK_METHOD2(authenticate, void(
+ Duration minTimeout,
+ Duration maxTimeout));
+
+ void unmocked_authenticate(
+ Duration minTimeout,
+ Duration maxTimeout);
+
MOCK_METHOD1(removeFramework, void(
slave::Framework* framework));
[mesos] 01/02: Made authentication tests independent of wall clock
time.
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gaston pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit da69a034cf2a257ff2ef39f17396f3dc7810a7b1
Author: Meng Zhu <mz...@mesosphere.io>
AuthorDate: Thu Sep 20 14:54:32 2018 -0700
Made authentication tests independent of wall clock time.
This patch pauses and manipulates the clock for all
authentication tests. This makes the test less flaky-prone
and speeds up the test run time.
Review: https://reviews.apache.org/r/68414/
---
src/tests/authentication_tests.cpp | 132 +++++++++++++++++++++++++++++--------
1 file changed, 106 insertions(+), 26 deletions(-)
diff --git a/src/tests/authentication_tests.cpp b/src/tests/authentication_tests.cpp
index f7a2cf1..bbb7e10 100644
--- a/src/tests/authentication_tests.cpp
+++ b/src/tests/authentication_tests.cpp
@@ -30,8 +30,6 @@
#include "master/detector/standalone.hpp"
-#include "sched/constants.hpp"
-
#include "tests/mesos.hpp"
#include "tests/utils.hpp"
@@ -59,6 +57,8 @@ class AuthenticationTest : public MesosTest {};
// denied registration by the master.
TEST_F(AuthenticationTest, UnauthenticatedFramework)
{
+ Clock::pause();
+
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
@@ -84,6 +84,8 @@ TEST_F(AuthenticationTest, UnauthenticatedFramework)
// denied registration by the master, but not shut down.
TEST_F(AuthenticationTest, UnauthenticatedSlave)
{
+ Clock::pause();
+
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
@@ -105,23 +107,21 @@ TEST_F(AuthenticationTest, UnauthenticatedSlave)
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags);
ASSERT_SOME(slave);
+ // Advance the clock to trigger the first registration attempt.
+ // The initial registration delay is between [0, registration_backoff_factor].
+ Clock::advance(flags.registration_backoff_factor);
+ Clock::settle();
+
AWAIT_READY(registerSlaveMessage1);
Future<RegisterSlaveMessage> registerSlaveMessage2 =
FUTURE_PROTOBUF(RegisterSlaveMessage(), _, _);
// Advance the clock to trigger another registration attempt.
- Clock::pause();
Clock::advance(slave::REGISTER_RETRY_INTERVAL_MAX);
Clock::settle();
- Clock::resume();
AWAIT_READY(registerSlaveMessage2);
-
- // Settle to make sure neither `SlaveRegisteredMessage` nor
- // `ShutdownMessage` are sent.
- Clock::pause();
- Clock::settle();
}
@@ -129,6 +129,8 @@ TEST_F(AuthenticationTest, UnauthenticatedSlave)
// authentication disabled, it registers unauthenticated frameworks.
TEST_F(AuthenticationTest, DisableFrameworkAuthentication)
{
+ Clock::pause();
+
master::Flags flags = CreateMasterFlags();
flags.authenticate_frameworks = false; // Disable authentication.
@@ -158,6 +160,8 @@ TEST_F(AuthenticationTest, DisableFrameworkAuthentication)
// authentication disabled, it registers unauthenticated slaves.
TEST_F(AuthenticationTest, DisableSlaveAuthentication)
{
+ Clock::pause();
+
master::Flags flags = CreateMasterFlags();
flags.authenticate_agents = false; // Disable authentication.
@@ -175,6 +179,11 @@ TEST_F(AuthenticationTest, DisableSlaveAuthentication)
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), slaveFlags);
ASSERT_SOME(slave);
+ // Advance the clock to trigger registration attempt.
+ // The initial registration delay is between [0, registration_backoff_factor].
+ Clock::advance(slaveFlags.registration_backoff_factor);
+ Clock::settle();
+
// Slave should be able to get registered.
AWAIT_READY(slaveRegisteredMessage);
ASSERT_NE("", slaveRegisteredMessage->slave_id().value());
@@ -186,6 +195,8 @@ TEST_F(AuthenticationTest, DisableSlaveAuthentication)
// FrameworkInfo.principal than Credential.principal.
TEST_F(AuthenticationTest, MismatchedFrameworkInfoPrincipal)
{
+ Clock::pause();
+
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
@@ -216,6 +227,8 @@ TEST_F(AuthenticationTest, MismatchedFrameworkInfoPrincipal)
// when authentication is not required.
TEST_F(AuthenticationTest, DisabledFrameworkAuthenticationPrincipalMismatch)
{
+ Clock::pause();
+
master::Flags flags = CreateMasterFlags();
flags.authenticate_frameworks = false; // Authentication not required.
@@ -248,6 +261,8 @@ TEST_F(AuthenticationTest, DisabledFrameworkAuthenticationPrincipalMismatch)
// register.
TEST_F(AuthenticationTest, UnspecifiedFrameworkInfoPrincipal)
{
+ Clock::pause();
+
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
@@ -276,6 +291,8 @@ TEST_F(AuthenticationTest, UnspecifiedFrameworkInfoPrincipal)
// authentication disabled, it registers authenticated frameworks.
TEST_F(AuthenticationTest, AuthenticatedFramework)
{
+ Clock::pause();
+
master::Flags flags = CreateMasterFlags();
flags.authenticate_frameworks = false; // Disable authentication.
@@ -305,6 +322,8 @@ TEST_F(AuthenticationTest, AuthenticatedFramework)
// authentication disabled, it registers authenticated slaves.
TEST_F(AuthenticationTest, AuthenticatedSlave)
{
+ Clock::pause();
+
master::Flags flags = CreateMasterFlags();
flags.authenticate_agents = false; // Disable authentication.
@@ -320,6 +339,12 @@ TEST_F(AuthenticationTest, AuthenticatedSlave)
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get());
ASSERT_SOME(slave);
+ // Advance the clock to trigger authentication and registration attempt.
+ // The initial authentication/registration delay is between
+ // [0, registration_backoff_factor].
+ Clock::advance(slave::DEFAULT_REGISTRATION_BACKOFF_FACTOR);
+ Clock::settle();
+
// Slave should be able to get registered.
AWAIT_READY(slaveRegisteredMessage);
ASSERT_NE("", slaveRegisteredMessage->slave_id().value());
@@ -330,6 +355,8 @@ TEST_F(AuthenticationTest, AuthenticatedSlave)
// authentication when authenticate message is lost.
TEST_F(AuthenticationTest, RetryFrameworkAuthentication)
{
+ Clock::pause();
+
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
@@ -350,10 +377,9 @@ TEST_F(AuthenticationTest, RetryFrameworkAuthentication)
.WillOnce(FutureSatisfy(®istered));
// Advance the clock for the scheduler to retry.
- Clock::pause();
- Clock::advance(Seconds(5));
+ Clock::advance(
+ mesos::internal::scheduler::DEFAULT_AUTHENTICATION_TIMEOUT_MAX);
Clock::settle();
- Clock::resume();
// Scheduler should be able to get registered.
AWAIT_READY(registered);
@@ -367,6 +393,8 @@ TEST_F(AuthenticationTest, RetryFrameworkAuthentication)
// authentication when authenticate message is lost.
TEST_F(AuthenticationTest, RetrySlaveAuthentication)
{
+ Clock::pause();
+
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
@@ -378,16 +406,20 @@ TEST_F(AuthenticationTest, RetrySlaveAuthentication)
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get());
ASSERT_SOME(slave);
+ // Advance the clock to trigger authentication attempt.
+ // Currently the initial backoff is [0, registration_backoff_factor]
+ // for agents both with or without authentication. See MESOS-9173.
+ Clock::advance(slave::DEFAULT_REGISTRATION_BACKOFF_FACTOR);
+ Clock::settle();
+
AWAIT_READY(authenticateMessage);
Future<SlaveRegisteredMessage> slaveRegisteredMessage =
FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _);
// Advance the clock for the slave to retry.
- Clock::pause();
Clock::advance(slave::DEFAULT_AUTHENTICATION_TIMEOUT_MAX);
Clock::settle();
- Clock::resume();
// Slave should be able to get registered.
AWAIT_READY(slaveRegisteredMessage);
@@ -466,6 +498,8 @@ TEST_F(AuthenticationTest, MasterRetriedAuthenticationHandling)
// is lost.
TEST_F(AuthenticationTest, DropIntermediateSASLMessage)
{
+ Clock::pause();
+
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
@@ -489,10 +523,9 @@ TEST_F(AuthenticationTest, DropIntermediateSASLMessage)
.WillOnce(FutureSatisfy(®istered));
// Advance the clock for the scheduler to retry.
- Clock::pause();
- Clock::advance(Seconds(5));
+ Clock::advance(
+ mesos::internal::scheduler::DEFAULT_AUTHENTICATION_TIMEOUT_MAX);
Clock::settle();
- Clock::resume();
// Ensure another authentication attempt was made.
AWAIT_READY(authenticationCompletedMessage);
@@ -510,6 +543,8 @@ TEST_F(AuthenticationTest, DropIntermediateSASLMessage)
// is lost.
TEST_F(AuthenticationTest, DropIntermediateSASLMessageForSlave)
{
+ Clock::pause();
+
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
@@ -521,6 +556,12 @@ TEST_F(AuthenticationTest, DropIntermediateSASLMessageForSlave)
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get());
ASSERT_SOME(slave);
+ // Advance the clock to trigger authentication attempt.
+ // Currently the initial backoff is [0, registration_backoff_factor]
+ // for agents both with or without authentication. See MESOS-9173.
+ Clock::advance(slave::DEFAULT_REGISTRATION_BACKOFF_FACTOR);
+ Clock::settle();
+
AWAIT_READY(authenticationStepMessage);
Future<AuthenticationCompletedMessage> authenticationCompletedMessage =
@@ -530,10 +571,8 @@ TEST_F(AuthenticationTest, DropIntermediateSASLMessageForSlave)
FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _);
// Advance the clock for the slave to retry.
- Clock::pause();
- Clock::advance(Seconds(5));
+ Clock::advance(slave::DEFAULT_AUTHENTICATION_TIMEOUT_MAX);
Clock::settle();
- Clock::resume();
// Ensure another authentication attempt was made.
AWAIT_READY(authenticationCompletedMessage);
@@ -552,6 +591,8 @@ TEST_F(AuthenticationTest, DropIntermediateSASLMessageForSlave)
// eventually register.
TEST_F(AuthenticationTest, DropFinalSASLMessage)
{
+ Clock::pause();
+
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
@@ -575,10 +616,9 @@ TEST_F(AuthenticationTest, DropFinalSASLMessage)
.WillOnce(FutureSatisfy(®istered));
// Advance the clock for the scheduler to retry.
- Clock::pause();
- Clock::advance(Seconds(5));
+ Clock::advance(
+ mesos::internal::scheduler::DEFAULT_AUTHENTICATION_TIMEOUT_MAX);
Clock::settle();
- Clock::resume();
// Ensure another authentication attempt was made.
AWAIT_READY(authenticationCompletedMessage);
@@ -599,6 +639,8 @@ TEST_F(AuthenticationTest, DropFinalSASLMessage)
// eventually register.
TEST_F(AuthenticationTest, DropFinalSASLMessageForSlave)
{
+ Clock::pause();
+
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
@@ -610,6 +652,12 @@ TEST_F(AuthenticationTest, DropFinalSASLMessageForSlave)
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get());
ASSERT_SOME(slave);
+ // Advance the clock to trigger authentication attempt.
+ // Currently the initial backoff is [0, registration_backoff_factor]
+ // for agents both with or without authentication. See MESOS-9173.
+ Clock::advance(slave::DEFAULT_REGISTRATION_BACKOFF_FACTOR);
+ Clock::settle();
+
AWAIT_READY(authenticationCompletedMessage);
authenticationCompletedMessage =
@@ -619,10 +667,8 @@ TEST_F(AuthenticationTest, DropFinalSASLMessageForSlave)
FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _);
// Advance the clock for the scheduler to retry.
- Clock::pause();
- Clock::advance(Seconds(5));
+ Clock::advance(slave::DEFAULT_AUTHENTICATION_TIMEOUT_MAX);
Clock::settle();
- Clock::resume();
// Ensure another authentication attempt was made.
AWAIT_READY(authenticationCompletedMessage);
@@ -638,6 +684,8 @@ TEST_F(AuthenticationTest, DropFinalSASLMessageForSlave)
// authenticates.
TEST_F(AuthenticationTest, MasterFailover)
{
+ Clock::pause();
+
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
@@ -680,6 +728,8 @@ TEST_F(AuthenticationTest, MasterFailover)
// authenticates.
TEST_F(AuthenticationTest, MasterFailoverDuringSlaveAuthentication)
{
+ Clock::pause();
+
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
@@ -692,6 +742,12 @@ TEST_F(AuthenticationTest, MasterFailoverDuringSlaveAuthentication)
Try<Owned<cluster::Slave>> slave = StartSlave(&detector, slaveFlags);
ASSERT_SOME(slave);
+ // Advance the clock to trigger authentication attempt.
+ // Currently the initial backoff is [0, registration_backoff_factor]
+ // for agents both with or without authentication. See MESOS-9173.
+ Clock::advance(slave::DEFAULT_REGISTRATION_BACKOFF_FACTOR);
+ Clock::settle();
+
AWAIT_READY(authenticateMessage);
// While the authentication is in progress simulate a failed over
@@ -706,6 +762,11 @@ TEST_F(AuthenticationTest, MasterFailoverDuringSlaveAuthentication)
// Appoint a new master and inform the slave about it.
detector.appoint(master.get()->pid);
+ // Advance the clock to trigger authentication and registration attempt.
+ // The initial registration delay is between [0, registration_backoff_factor].
+ Clock::advance(slave::DEFAULT_REGISTRATION_BACKOFF_FACTOR);
+ Clock::settle();
+
// Slave should be able to get registered.
AWAIT_READY(slaveRegisteredMessage);
ASSERT_NE("", slaveRegisteredMessage->slave_id().value());
@@ -717,6 +778,8 @@ TEST_F(AuthenticationTest, MasterFailoverDuringSlaveAuthentication)
// detected due to leader election), it is handled properly.
TEST_F(AuthenticationTest, LeaderElection)
{
+ Clock::pause();
+
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
@@ -754,6 +817,8 @@ TEST_F(AuthenticationTest, LeaderElection)
// detected due to leader election), it is handled properly.
TEST_F(AuthenticationTest, LeaderElectionDuringSlaveAuthentication)
{
+ Clock::pause();
+
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
@@ -766,6 +831,12 @@ TEST_F(AuthenticationTest, LeaderElectionDuringSlaveAuthentication)
Try<Owned<cluster::Slave>> slave = StartSlave(&detector, slaveFlags);
ASSERT_SOME(slave);
+ // Advance the clock to trigger authentication attempt.
+ // Currently the initial backoff is [0, registration_backoff_factor]
+ // for agents both with or without authentication. See MESOS-9173.
+ Clock::advance(slave::DEFAULT_REGISTRATION_BACKOFF_FACTOR);
+ Clock::settle();
+
// Drop the intermediate SASL message so that authentication fails.
AWAIT_READY(authenticationStepMessage);
@@ -775,6 +846,11 @@ TEST_F(AuthenticationTest, LeaderElectionDuringSlaveAuthentication)
// Appoint a new master and inform the slave about it.
detector.appoint(master.get()->pid);
+ // Advance the clock to trigger authentication and registration attempt.
+ // The initial registration delay is between [0, registration_backoff_factor].
+ Clock::advance(slave::DEFAULT_REGISTRATION_BACKOFF_FACTOR);
+ Clock::settle();
+
// Slave should be able to get registered.
AWAIT_READY(slaveRegisteredMessage);
ASSERT_NE("", slaveRegisteredMessage->slave_id().value());
@@ -786,6 +862,8 @@ TEST_F(AuthenticationTest, LeaderElectionDuringSlaveAuthentication)
// with the master when it comes back up.
TEST_F(AuthenticationTest, SchedulerFailover)
{
+ Clock::pause();
+
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
@@ -853,6 +931,8 @@ TEST_F(AuthenticationTest, SchedulerFailover)
// authentication.
TEST_F(AuthenticationTest, RejectedSchedulerFailover)
{
+ Clock::pause();
+
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);