You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by bm...@apache.org on 2018/08/16 07:33:15 UTC
[mesos] branch 1.4.x updated (612ec2c -> 844e405)
This is an automated email from the ASF dual-hosted git repository.
bmahler pushed a change to branch 1.4.x
in repository https://gitbox.apache.org/repos/asf/mesos.git.
from 612ec2c Updated Git repository URLs.
new 58d8e48 Increased and added flag for the master's authentication timeout.
new bf3694e Added MESOS-9145 to the 1.4.3 CHANGELOG.
new 746688d Fixed an authentication request amplification issue in the master.
new 844e405 Added MESOS-9144 to the 1.4.3 CHANGELOG.
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGELOG | 9 +++++++++
docs/authentication.md | 4 ++++
src/master/constants.hpp | 8 ++++++++
src/master/flags.cpp | 10 ++++++++++
src/master/flags.hpp | 1 +
src/master/master.cpp | 38 +++++++++++++++++++++-----------------
6 files changed, 53 insertions(+), 17 deletions(-)
[mesos] 03/04: Fixed an authentication request amplification issue
in the master.
Posted by bm...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bmahler pushed a commit to branch 1.4.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 746688daa1db65de6d750d8d8d21db33ee34b6e1
Author: Benjamin Mahler <bm...@apache.org>
AuthorDate: Fri Aug 10 17:14:05 2018 -0700
Fixed an authentication request amplification issue in the master.
Per MESOS-9144, re-enqueuing authentication requests leads to an
amplification effect which can overwhelm the master if requests
continue to arrive rapidly on a heavily loaded master. This patch
avoids the re-enqueing and ensures the master immediately processes
a new authentication request for a client.
Review: https://reviews.apache.org/r/68306
---
src/master/master.cpp | 36 ++++++++++++++++++++----------------
1 file changed, 20 insertions(+), 16 deletions(-)
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 8256999..d8dcfc1 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -7849,7 +7849,7 @@ void Master::authenticate(const UPID& from, const UPID& pid)
// about this discrepancy via ping messages so that it can
// re-register.
- authenticated.erase(pid);
+ bool erased = authenticated.erase(pid) > 0;
if (authenticator.isNone()) {
// The default authenticator is CRAM-MD5 rather than none.
@@ -7872,30 +7872,28 @@ void Master::authenticate(const UPID& from, const UPID& pid)
return;
}
+ // If a new authentication is occurring for a client that already
+ // has an authentication in progress, we discard the old one
+ // (since the client is no longer interested in it) and
+ // immediately proceed with the new authentication.
if (authenticating.contains(pid)) {
- LOG(INFO) << "Queuing up authentication request from " << pid
- << " because authentication is still in progress";
-
- // Try to cancel the in progress authentication by discarding the
- // future.
- authenticating[pid].discard();
-
- // Retry after the current authenticator session finishes.
- authenticating[pid]
- .onAny(defer(self(), &Self::authenticate, from, pid));
+ authenticating.at(pid).discard();
+ authenticating.erase(pid);
- return;
+ LOG(INFO) << "Re-authenticating " << pid << ";"
+ << " discarding outstanding authentication";
+ } else {
+ LOG(INFO) << "Authenticating " << pid
+ << (erased ? "; clearing previous authentication" : "");
}
- LOG(INFO) << "Authenticating " << pid;
-
// Start authentication.
const Future<Option<string>> future = authenticator.get()->authenticate(from);
// Save our state.
authenticating[pid] = future;
- future.onAny(defer(self(), &Self::_authenticate, pid, lambda::_1));
+ future.onAny(defer(self(), &Self::_authenticate, pid, future));
// Don't wait for authentication to complete forever.
delay(flags.authentication_v0_timeout,
@@ -7909,6 +7907,13 @@ void Master::_authenticate(
const UPID& pid,
const Future<Option<string>>& future)
{
+ // Ignore stale authentication results (if the authentication
+ // future has been overwritten).
+ if (authenticating.get(pid) != future) {
+ LOG(INFO) << "Ignoring stale authentication result of " << pid;
+ return;
+ }
+
if (future.isReady() && future->isSome()) {
LOG(INFO) << "Successfully authenticated principal '" << future->get()
<< "' at " << pid;
@@ -7924,7 +7929,6 @@ void Master::_authenticate(
LOG(INFO) << "Authentication of " << pid << " was discarded";
}
- CHECK(authenticating.contains(pid));
authenticating.erase(pid);
}
[mesos] 04/04: Added MESOS-9144 to the 1.4.3 CHANGELOG.
Posted by bm...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bmahler pushed a commit to branch 1.4.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 844e4054daf9857e29f04618811671869c0c8937
Author: Benjamin Mahler <bm...@apache.org>
AuthorDate: Wed Aug 15 16:24:10 2018 -0700
Added MESOS-9144 to the 1.4.3 CHANGELOG.
---
CHANGELOG | 1 +
1 file changed, 1 insertion(+)
diff --git a/CHANGELOG b/CHANGELOG
index dfedd05..acb4145 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -3,6 +3,7 @@ Release Notes - Mesos - Version 1.4.3 (WIP)
* This is a bug fix release.
** Bug
+ * [MESOS-9144] - Master authentication handling leads to request amplification.
* [MESOS-9145] - Master has a fragile burned-in 5s authentication timeout.
[mesos] 02/04: Added MESOS-9145 to the 1.4.3 CHANGELOG.
Posted by bm...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bmahler pushed a commit to branch 1.4.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit bf3694ef44cbf1e73c1afbfec1ed54a14eec08be
Author: Benjamin Mahler <bm...@apache.org>
AuthorDate: Wed Aug 15 16:23:49 2018 -0700
Added MESOS-9145 to the 1.4.3 CHANGELOG.
---
CHANGELOG | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/CHANGELOG b/CHANGELOG
index 7ea7924..dfedd05 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,11 @@
+Release Notes - Mesos - Version 1.4.3 (WIP)
+-------------------------------------------
+* This is a bug fix release.
+
+** Bug
+ * [MESOS-9145] - Master has a fragile burned-in 5s authentication timeout.
+
+
Release Notes - Mesos - Version 1.4.2
-------------------------------------------
* This is a bug fix release.
[mesos] 01/04: Increased and added flag for the master's
authentication timeout.
Posted by bm...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bmahler pushed a commit to branch 1.4.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 58d8e4804408eb112b537609fa60f188f5fed248
Author: Benjamin Mahler <bm...@apache.org>
AuthorDate: Fri Aug 10 17:09:52 2018 -0700
Increased and added flag for the master's authentication timeout.
There is not a lot of value in the master timing out a client's
authentication, other than releasing a small amount of resources.
We currently have a burned in 5 second timeout, which is largely
sufficient since most authenticators are implemented to use an
actor per session and avoid any head-of-line blocking.
Ideally, the master would know how long the client's timeout and
the master can use that for its own timeout. The current max backoff
for schedulers and agents is 1 minute, so this patch bumps the
master's timeout to be closer to that (15 seconds). We don't bump it
further because the vast majority of the timeout time is spent in
the initial trip through the master's queue, which occurs before
the master sets up its timeout.
This also adds a flag, both to allow users to tune this, as well
as to allow us to control timing in tests.
Review: https://reviews.apache.org/r/68305
---
docs/authentication.md | 4 ++++
src/master/constants.hpp | 8 ++++++++
src/master/flags.cpp | 10 ++++++++++
src/master/flags.hpp | 1 +
src/master/master.cpp | 2 +-
5 files changed, 24 insertions(+), 1 deletion(-)
diff --git a/docs/authentication.md b/docs/authentication.md
index a96de6e..1b00695 100644
--- a/docs/authentication.md
+++ b/docs/authentication.md
@@ -64,6 +64,10 @@ Mesos master and agent processes. For more information, refer to the
allowed to register. If `false` (the default), unauthenticated agents are also
allowed to register.
+* `--authentication_v0_timeout` - The timeout within which an authentication is
+ expected to complete against a v0 framework or agent. This does not apply to
+ the v0 or v1 HTTP APIs.(default: `15secs`)
+
* `--authenticators` - Specifies which authenticator module to use. The default
is `crammd5`, but additional modules can be added using the `--modules`
option.
diff --git a/src/master/constants.hpp b/src/master/constants.hpp
index c35ed45..e8a8085 100644
--- a/src/master/constants.hpp
+++ b/src/master/constants.hpp
@@ -45,6 +45,14 @@ constexpr double MIN_CPUS = 0.01;
// Minimum amount of memory per offer.
constexpr Bytes MIN_MEM = Megabytes(32);
+// Default timeout for v0 framework and agent authentication
+// before the master cancels an in-progress authentication.
+//
+// TODO(bmahler): Ideally, we remove this v0-style authentication
+// in favor of just using HTTP authentication at the libprocess
+// layer.
+constexpr Duration DEFAULT_AUTHENTICATION_V0_TIMEOUT = Seconds(15);
+
// Default interval the master uses to send heartbeats to an HTTP
// scheduler.
constexpr Duration DEFAULT_HEARTBEAT_INTERVAL = Seconds(15);
diff --git a/src/master/flags.cpp b/src/master/flags.cpp
index af0014c..19145db 100644
--- a/src/master/flags.cpp
+++ b/src/master/flags.cpp
@@ -231,6 +231,16 @@ mesos::internal::master::Flags::Flags()
"If `false`, unauthenticated agents are also allowed to register.",
false);
+ // TODO(bmahler): Ideally, we remove this v0-style authentication
+ // in favor of just using HTTP authentication at the libprocess
+ // layer.
+ add(&Flags::authentication_v0_timeout,
+ "authentication_v0_timeout",
+ "The timeout within which an authentication is expected\n"
+ "to complete against a v0 framework or agent. This does not\n"
+ "apply to the v0 or v1 HTTP APIs.",
+ DEFAULT_AUTHENTICATION_V0_TIMEOUT);
+
// TODO(zhitao): Remove deprecated `--authenticate_http` flag name after
// the deprecation cycle which started with Mesos 1.0.
add(&Flags::authenticate_http_readwrite,
diff --git a/src/master/flags.hpp b/src/master/flags.hpp
index b262fd2..2e659df 100644
--- a/src/master/flags.hpp
+++ b/src/master/flags.hpp
@@ -68,6 +68,7 @@ public:
Option<std::string> weights;
bool authenticate_frameworks;
bool authenticate_agents;
+ Duration authentication_v0_timeout;
bool authenticate_http_readonly;
bool authenticate_http_readwrite;
bool authenticate_http_frameworks;
diff --git a/src/master/master.cpp b/src/master/master.cpp
index f654300..8256999 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -7898,7 +7898,7 @@ void Master::authenticate(const UPID& from, const UPID& pid)
future.onAny(defer(self(), &Self::_authenticate, pid, lambda::_1));
// Don't wait for authentication to complete forever.
- delay(Seconds(5),
+ delay(flags.authentication_v0_timeout,
self(),
&Self::authenticationTimeout,
future);