You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by bm...@apache.org on 2016/03/03 06:30:51 UTC
[1/4] mesos git commit: Updated tests now that /metrics/snapshot is
not rate limited.
Repository: mesos
Updated Branches:
refs/heads/master fb8a60cde -> 4dc863ca9
Updated tests now that /metrics/snapshot is not rate limited.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/4dc863ca
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/4dc863ca
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/4dc863ca
Branch: refs/heads/master
Commit: 4dc863ca95d899845a9476fc406f411396b006d4
Parents: 0633252
Author: Benjamin Mahler <bm...@apache.org>
Authored: Wed Mar 2 21:23:14 2016 -0800
Committer: Benjamin Mahler <bm...@apache.org>
Committed: Wed Mar 2 21:24:39 2016 -0800
----------------------------------------------------------------------
src/tests/rate_limiting_tests.cpp | 77 +++-------------------------------
1 file changed, 6 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/4dc863ca/src/tests/rate_limiting_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/rate_limiting_tests.cpp b/src/tests/rate_limiting_tests.cpp
index caced73..352a5e7 100644
--- a/src/tests/rate_limiting_tests.cpp
+++ b/src/tests/rate_limiting_tests.cpp
@@ -101,12 +101,6 @@ TEST_F(RateLimitingTest, NoRateLimiting)
// '_recover()' completes.
Clock::settle();
- // Advance before the test so that the 1st call to Metrics endpoint
- // is not throttled. MetricsProcess which hosts the endpoint
- // throttles requests at 2qps and its singleton instance is shared
- // across tests.
- Clock::advance(Milliseconds(501));
-
// Message counters not present before the framework registers.
{
JSON::Object metrics = Metrics();
@@ -145,9 +139,6 @@ TEST_F(RateLimitingTest, NoRateLimiting)
const process::UPID schedulerPid = frameworkRegisteredMessage.get().to;
- // For metrics endpoint.
- Clock::advance(Milliseconds(501));
-
// Send a duplicate subscribe call. Master sends
// FrameworkRegisteredMessage back after processing it.
{
@@ -191,9 +182,6 @@ TEST_F(RateLimitingTest, NoRateLimiting)
// unlimited rate.
AWAIT_READY(removeFramework);
- // For metrics endpoint.
- Clock::advance(Milliseconds(501));
-
// Message counters removed after the framework is unregistered.
{
JSON::Object metrics = Metrics();
@@ -226,12 +214,6 @@ TEST_F(RateLimitingTest, RateLimitingEnabled)
// '_recover()' completes.
Clock::settle();
- // Advance before the test so that the 1st call to Metrics endpoint
- // is not throttled. MetricsProcess which hosts the endpoint
- // throttles requests at 2qps and its singleton instance is shared
- // across tests.
- Clock::advance(Milliseconds(501));
-
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
@@ -294,10 +276,6 @@ TEST_F(RateLimitingTest, RateLimitingEnabled)
process::post(schedulerPid, master.get(), subscribeCall.get());
- // Advance for half a second and verify that the message is still
- // not processed.
- Clock::advance(Milliseconds(501));
-
// Settle to make sure all events not delayed are processed.
Clock::settle();
@@ -322,8 +300,8 @@ TEST_F(RateLimitingTest, RateLimitingEnabled)
EXPECT_TRUE(duplicateFrameworkRegisteredMessage.isPending());
}
- // After another half a second the message should be processed.
- Clock::advance(Milliseconds(501));
+ // After a second the message should be processed.
+ Clock::advance(Seconds(1));
AWAIT_READY(duplicateFrameworkRegisteredMessage);
// Verify counters after processing of the message.
@@ -376,12 +354,6 @@ TEST_F(RateLimitingTest, DifferentPrincipalFrameworks)
// '_recover()' completes.
Clock::settle();
- // Advance before the test so that the 1st call to Metrics endpoint
- // is not throttled. MetricsProcess which hosts the endpoint
- // throttles requests at 2qps and its singleton instance is shared
- // across tests.
- Clock::advance(Milliseconds(501));
-
// 1. Register two frameworks.
// 1.1. Create the first framework.
@@ -582,9 +554,6 @@ TEST_F(RateLimitingTest, DifferentPrincipalFrameworks)
// the metrics endpoint query.
Clock::settle();
- // Advance for Metrics rate limiting.
- Clock::advance(Milliseconds(501));
-
JSON::Object metrics = Metrics();
EXPECT_EQ(
@@ -617,12 +586,6 @@ TEST_F(RateLimitingTest, SamePrincipalFrameworks)
// '_recover()' completes.
Clock::settle();
- // Advance before the test so that the 1st call to Metrics endpoint
- // is not throttled. MetricsProcess which hosts the endpoint
- // throttles requests at 2qps and its singleton instance is shared
- // across tests.
- Clock::advance(Milliseconds(501));
-
// 1. Register two frameworks.
// 1.1. Create the first framework.
@@ -709,9 +672,6 @@ TEST_F(RateLimitingTest, SamePrincipalFrameworks)
Clock::settle();
EXPECT_TRUE(duplicateFrameworkRegisteredMessage2.isPending());
- // For metrics endpoint.
- Clock::advance(Milliseconds(501));
-
{
JSON::Object metrics = Metrics();
@@ -731,9 +691,8 @@ TEST_F(RateLimitingTest, SamePrincipalFrameworks)
metrics.values[messages_processed].as<JSON::Number>().as<int64_t>());
}
- // Advance for another half a second to make sure throttled
- // message is processed.
- Clock::advance(Milliseconds(501));
+ // Advance for a second to make sure throttled message is processed.
+ Clock::advance(Seconds(1));
AWAIT_READY(duplicateFrameworkRegisteredMessage2);
@@ -753,9 +712,6 @@ TEST_F(RateLimitingTest, SamePrincipalFrameworks)
// Message counters are not removed after the first framework is
// unregistered.
- // For metrics endpoint.
- Clock::advance(Milliseconds(501));
-
{
JSON::Object metrics = Metrics();
@@ -789,12 +745,6 @@ TEST_F(RateLimitingTest, SchedulerFailover)
// '_recover()' completes.
Clock::settle();
- // Advance before the test so that the 1st call to Metrics endpoint
- // is not throttled. MetricsProcess which hosts the endpoint
- // throttles requests at 2qps and its singleton instance is shared
- // across tests.
- Clock::advance(Milliseconds(501));
-
// 1. Launch the first (i.e., failing) scheduler and verify its
// counters.
@@ -909,9 +859,6 @@ TEST_F(RateLimitingTest, SchedulerFailover)
// throttling the new scheduler instance.
EXPECT_TRUE(duplicateFrameworkRegisteredMessage.isPending());
- // Advance for metrics.
- Clock::advance(Milliseconds(501));
-
{
JSON::Object metrics = Metrics();
@@ -932,14 +879,11 @@ TEST_F(RateLimitingTest, SchedulerFailover)
metrics.values[messages_processed].as<JSON::Number>().as<int64_t>());
}
- // Need another half a second to have it processed.
- Clock::advance(Milliseconds(501));
+ // Need a second to have it processed.
+ Clock::advance(Seconds(1));
AWAIT_READY(duplicateFrameworkRegisteredMessage);
- // Advance for metrics.
- Clock::advance(Milliseconds(501));
-
{
JSON::Object metrics = Metrics();
@@ -985,12 +929,6 @@ TEST_F(RateLimitingTest, CapacityReached)
Clock::pause();
- // Advance before the test so that the 1st call to Metrics endpoint
- // is not throttled. MetricsProcess which hosts the endpoint
- // throttles requests at 2qps and its singleton instance is shared
- // across tests.
- Clock::advance(Milliseconds(501));
-
MockScheduler sched;
FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
@@ -1089,9 +1027,6 @@ TEST_F(RateLimitingTest, CapacityReached)
EXPECT_EQ(DRIVER_STOPPED, driver->join());
delete driver;
- // Wait for half a second for metrics endpoint.
- Clock::advance(Milliseconds(501));
-
{
JSON::Object metrics = Metrics();
[4/4] mesos git commit: Updated /metrics/snapshot rate limiting to be
configurable.
Posted by bm...@apache.org.
Updated /metrics/snapshot rate limiting to be configurable.
Previously, the /metrics/snapshot endpoint had a hard-coded rate
limit of 2 requests per second. This adds an environment variable
`LIBPROCESS_METRICS_SNAPSHOT_ENDPOINT_RATE_LIMIT` for changing
or disabling the rate limit. Note that the existing hard-coded
rate limit remains the default.
Review: https://reviews.apache.org/r/44070/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/9da6e143
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/9da6e143
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/9da6e143
Branch: refs/heads/master
Commit: 9da6e143e990dba1d1857429e9bf89905529e36e
Parents: fb8a60c
Author: Benjamin Bannier <be...@mesosphere.io>
Authored: Wed Mar 2 18:09:00 2016 -0800
Committer: Benjamin Mahler <bm...@apache.org>
Committed: Wed Mar 2 21:24:39 2016 -0800
----------------------------------------------------------------------
.../include/process/metrics/metrics.hpp | 14 +++-
3rdparty/libprocess/src/metrics/metrics.cpp | 88 ++++++++++++++++++--
3rdparty/libprocess/src/process.cpp | 9 +-
3 files changed, 91 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/9da6e143/3rdparty/libprocess/include/process/metrics/metrics.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/include/process/metrics/metrics.hpp b/3rdparty/libprocess/include/process/metrics/metrics.hpp
index 09b716b..7abfadf 100644
--- a/3rdparty/libprocess/include/process/metrics/metrics.hpp
+++ b/3rdparty/libprocess/include/process/metrics/metrics.hpp
@@ -30,6 +30,9 @@
namespace process {
namespace metrics {
+// Initializes the metrics library.
+void initialize();
+
namespace internal {
class MetricsProcess : public Process<MetricsProcess>
@@ -47,9 +50,9 @@ protected:
private:
static std::string help();
- MetricsProcess()
+ MetricsProcess(const Option<Owned<RateLimiter>>& _limiter)
: ProcessBase("metrics"),
- limiter(2, Seconds(1))
+ limiter(_limiter)
{}
// Non-copyable, non-assignable.
@@ -69,8 +72,11 @@ private:
// The Owned<Metric> is an explicit copy of the Metric passed to 'add'.
hashmap<std::string, Owned<Metric> > metrics;
- // Used to rate limit the endpoint.
- RateLimiter limiter;
+ // Used to rate limit the snapshot endpoint.
+ Option<Owned<RateLimiter>> limiter;
+
+ // Needed for access to the private constructor.
+ friend void process::metrics::initialize();
};
} // namespace internal {
http://git-wip-us.apache.org/repos/asf/mesos/blob/9da6e143/3rdparty/libprocess/src/metrics/metrics.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/metrics/metrics.cpp b/3rdparty/libprocess/src/metrics/metrics.cpp
index a984008..8812fe2 100644
--- a/3rdparty/libprocess/src/metrics/metrics.cpp
+++ b/3rdparty/libprocess/src/metrics/metrics.cpp
@@ -14,26 +14,37 @@
#include <list>
#include <string>
+#include <vector>
#include <process/collect.hpp>
#include <process/dispatch.hpp>
#include <process/help.hpp>
#include <process/once.hpp>
+#include <process/owned.hpp>
#include <process/process.hpp>
#include <process/metrics/metrics.hpp>
+#include <stout/duration.hpp>
+#include <stout/error.hpp>
#include <stout/foreach.hpp>
#include <stout/hashmap.hpp>
+#include <stout/numify.hpp>
+#include <stout/option.hpp>
+#include <stout/os.hpp>
using std::list;
using std::string;
+using std::vector;
namespace process {
namespace metrics {
-namespace internal {
-MetricsProcess* MetricsProcess::instance()
+
+static internal::MetricsProcess* metrics_process = NULL;
+
+
+void initialize()
{
// To prevent a deadlock, we must ensure libprocess is
// initialized. Otherwise, libprocess will be implicitly
@@ -43,16 +54,69 @@ MetricsProcess* MetricsProcess::instance()
// 'done()' to ever be called.
process::initialize();
- static MetricsProcess* singleton = NULL;
static Once* initialized = new Once();
-
if (!initialized->once()) {
- singleton = new MetricsProcess();
- spawn(singleton);
+ Option<string> limit =
+ os::getenv("LIBPROCESS_METRICS_SNAPSHOT_ENDPOINT_RATE_LIMIT");
+
+ Option<Owned<RateLimiter>> limiter;
+
+ // By default, we apply a rate limit of 2 requests
+ // per second to the metrics snapshot endpoint in
+ // order to maintain backwards compatibility (before
+ // this was made configurable, we hard-coded a limit
+ // of 2 requests per second).
+ if (limit.isNone()) {
+ limiter = Owned<RateLimiter>(new RateLimiter(2, Seconds(1)));
+ } else if (limit->empty()) {
+ limiter = None();
+ } else {
+ // TODO(vinod): Move this parsing logic to flags
+ // once we have a 'Rate' abstraction in stout.
+ Option<Error> reason;
+ vector<string> tokens = strings::tokenize(limit.get(), "/");
+
+ if (tokens.size() == 2) {
+ Try<int> requests = numify<int>(tokens[0]);
+ Try<Duration> interval = Duration::parse(tokens[1]);
+
+ if (requests.isError()) {
+ reason = Error(
+ "Failed to parse the number of requests: " + requests.error());
+ } else if (interval.isError()) {
+ reason = Error(
+ "Failed to parse the interval: " + interval.error());
+ } else {
+ limiter = Owned<RateLimiter>(
+ new RateLimiter(requests.get(), interval.get()));
+ }
+ }
+
+ if (limiter.isNone()) {
+ EXIT(EXIT_FAILURE)
+ << "Failed to parse LIBPROCESS_METRICS_SNAPSHOT_ENDPOINT_RATE_LIMIT "
+ << "'" << limit.get() << "'"
+ << " (format is <number of requests>/<interval duration>)"
+ << (reason.isSome() ? ": " + reason.get().message : "");
+ }
+ }
+
+ metrics_process = new internal::MetricsProcess(limiter);
+ spawn(metrics_process);
+
initialized->done();
}
+}
+
- return singleton;
+namespace internal {
+
+
+MetricsProcess* MetricsProcess::instance()
+{
+ metrics::initialize();
+
+ return metrics_process;
}
@@ -103,8 +167,13 @@ Future<Nothing> MetricsProcess::remove(const string& name)
Future<http::Response> MetricsProcess::snapshot(const http::Request& request)
{
- return limiter.acquire()
- .then(defer(self(), &Self::_snapshot, request));
+ Future<Nothing> acquire = Nothing();
+
+ if (limiter.isSome()) {
+ acquire = limiter.get()->acquire();
+ }
+
+ return acquire.then(defer(self(), &Self::_snapshot, request));
}
@@ -194,5 +263,6 @@ Future<http::Response> MetricsProcess::__snapshot(
}
} // namespace internal {
+
} // namespace metrics {
} // namespace process {
http://git-wip-us.apache.org/repos/asf/mesos/blob/9da6e143/3rdparty/libprocess/src/process.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/process.cpp b/3rdparty/libprocess/src/process.cpp
index 4a53020..5dae1e7 100644
--- a/3rdparty/libprocess/src/process.cpp
+++ b/3rdparty/libprocess/src/process.cpp
@@ -971,13 +971,8 @@ void initialize(const string& delegate)
// Create the global HTTP authentication router.
authenticator_manager = new AuthenticatorManager();
- // Ensure metrics process is running.
- // TODO(bmahler): Consider initializing this consistently with
- // the other global Processes.
- metrics::internal::MetricsProcess* metricsProcess =
- metrics::internal::MetricsProcess::instance();
-
- CHECK_NOTNULL(metricsProcess);
+ // Initialize the metrics process.
+ metrics::initialize();
// Initialize the mime types.
mime::initialize();
[2/4] mesos git commit: Documented
LIBPROCESS_METRICS_SNAPSHOT_ENDPOINT_RATE_LIMIT.
Posted by bm...@apache.org.
Documented LIBPROCESS_METRICS_SNAPSHOT_ENDPOINT_RATE_LIMIT.
This environment variable allows the user to configure the rate
limit for the /metrics/snapshot endpoint.
Review: https://reviews.apache.org/r/44071/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/b86741dd
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/b86741dd
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/b86741dd
Branch: refs/heads/master
Commit: b86741ddf1618483071667c10a531444adbb0cb4
Parents: 9da6e14
Author: Benjamin Bannier <be...@mesosphere.io>
Authored: Wed Mar 2 20:12:57 2016 -0800
Committer: Benjamin Mahler <bm...@apache.org>
Committed: Wed Mar 2 21:24:39 2016 -0800
----------------------------------------------------------------------
docs/configuration.md | 11 +++++++++++
1 file changed, 11 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/b86741dd/docs/configuration.md
----------------------------------------------------------------------
diff --git a/docs/configuration.md b/docs/configuration.md
index b597479..305ba2c 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -1698,6 +1698,17 @@ isolator. (default: false)
provided separately.
</td>
</tr>
+ <tr>
+ <td>
+ LIBPROCESS_METRICS_SNAPSHOT_ENDPOINT_RATE_LIMIT
+ </td>
+ <td>
+ If set, this variable can be used to configure the rate limit
+ applied to the /metrics/snapshot endpoint. The format is
+ `<number of requests>/<interval duration>`.
+ Examples: `10/1secs`, `100/10secs`, etc.
+ </td>
+ </tr>
</table>
[3/4] mesos git commit: Disabled rate limiting of /metrics/snapshot
in the mesos tests.
Posted by bm...@apache.org.
Disabled rate limiting of /metrics/snapshot in the mesos tests.
Previously, rate limiting was enabled for /metrics/snapshot and so
tests that obtained metrics were prone to being slowed down, unless
manual clock manipulation was done in that test.
Review: https://reviews.apache.org/r/44073/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/06332520
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/06332520
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/06332520
Branch: refs/heads/master
Commit: 06332520b732cf806d9e1234d90a6715a8e980ed
Parents: b86741d
Author: Benjamin Bannier <be...@mesosphere.io>
Authored: Wed Mar 2 20:23:07 2016 -0800
Committer: Benjamin Mahler <bm...@apache.org>
Committed: Wed Mar 2 21:24:39 2016 -0800
----------------------------------------------------------------------
src/tests/main.cpp | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/06332520/src/tests/main.cpp
----------------------------------------------------------------------
diff --git a/src/tests/main.cpp b/src/tests/main.cpp
index 942488e..1425850 100644
--- a/src/tests/main.cpp
+++ b/src/tests/main.cpp
@@ -70,6 +70,10 @@ int main(int argc, char** argv)
return EXIT_FAILURE;
}
+ // Disable /metrics/snapshot rate limiting, but do not
+ // overwrite whatever the user set.
+ os::setenv("LIBPROCESS_METRICS_SNAPSHOT_ENDPOINT_RATE_LIMIT", "", false);
+
// Initialize libprocess.
process::initialize();