You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by bm...@apache.org on 2015/05/08 05:23:43 UTC
[1/3] mesos git commit: Slowed the webui polling for larger clusters.
Repository: mesos
Updated Branches:
refs/heads/master d37889da1 -> 115db9250
Slowed the webui polling for larger clusters.
Review: https://reviews.apache.org/r/33933
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/389995dc
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/389995dc
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/389995dc
Branch: refs/heads/master
Commit: 389995dcc76e18a9e9e72c1da58410438112a487
Parents: d37889d
Author: Benjamin Mahler <be...@gmail.com>
Authored: Wed May 6 22:12:35 2015 -0700
Committer: Benjamin Mahler <be...@gmail.com>
Committed: Thu May 7 20:04:00 2015 -0700
----------------------------------------------------------------------
src/webui/master/static/js/controllers.js | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/389995dc/src/webui/master/static/js/controllers.js
----------------------------------------------------------------------
diff --git a/src/webui/master/static/js/controllers.js b/src/webui/master/static/js/controllers.js
index 7ee3d3d..9384694 100644
--- a/src/webui/master/static/js/controllers.js
+++ b/src/webui/master/static/js/controllers.js
@@ -37,9 +37,15 @@
} else if (num_slaves < 1000) {
return 20000;
} else if (num_slaves < 5000) {
- return 30000;
- } else {
return 60000;
+ } else if (num_slaves < 10000) {
+ return 120000;
+ } else if (num_slaves < 15000) {
+ return 240000;
+ } else if (num_slaves < 20000) {
+ return 480000;
+ } else {
+ return 960000;
}
}
[3/3] mesos git commit: Improved HTTP request logging for
master/slave endpoints.
Posted by bm...@apache.org.
Improved HTTP request logging for master/slave endpoints.
Review: https://reviews.apache.org/r/33935
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/115db925
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/115db925
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/115db925
Branch: refs/heads/master
Commit: 115db9250e096edf70da54c3f6aea18aefeb1a06
Parents: 21ee7fa
Author: Benjamin Mahler <be...@gmail.com>
Authored: Wed May 6 22:18:39 2015 -0700
Committer: Benjamin Mahler <be...@gmail.com>
Committed: Thu May 7 20:11:30 2015 -0700
----------------------------------------------------------------------
src/master/http.cpp | 53 ++++++++++++++++++++++++----------------------
src/master/master.cpp | 49 +++++++++++++++++++++++++++++++++---------
src/master/master.hpp | 29 ++++++++++++++-----------
src/slave/http.cpp | 22 +++++++++++++++----
src/slave/slave.cpp | 17 ++++++++++++---
src/slave/slave.hpp | 10 ++++++---
6 files changed, 123 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/115db925/src/master/http.cpp
----------------------------------------------------------------------
diff --git a/src/master/http.cpp b/src/master/http.cpp
index f0668aa..acc85b6 100644
--- a/src/master/http.cpp
+++ b/src/master/http.cpp
@@ -254,6 +254,22 @@ JSON::Object model(const Role& role)
}
+void Master::Http::log(const Request& request)
+{
+ Option<string> userAgent = request.headers.get("User-Agent");
+ Option<string> forwardedFor = request.headers.get("X-Forwarded-For");
+
+ LOG(INFO) << "HTTP " << request.method << " for " << request.path
+ << " from " << request.client
+ << (userAgent.isSome()
+ ? " with User-Agent='" + userAgent.get() + "'"
+ : "")
+ << (forwardedFor.isSome()
+ ? " with X-Forwarded-For='" + forwardedFor.get() + "'"
+ : "");
+}
+
+
const string Master::Http::HEALTH_HELP = HELP(
TLDR(
"Health check of the Master."),
@@ -264,7 +280,7 @@ const string Master::Http::HEALTH_HELP = HELP(
"Delayed responses are also indicative of poor health."));
-Future<Response> Master::Http::health(const Request& request)
+Future<Response> Master::Http::health(const Request& request) const
{
return OK();
}
@@ -313,10 +329,8 @@ Try<string> getFormValue(
}
-Future<Response> Master::Http::observe(const Request& request)
+Future<Response> Master::Http::observe(const Request& request) const
{
- LOG(INFO) << "HTTP request for '" << request.path << "'";
-
Try<hashmap<string, string>> decode =
process::http::query::decode(request.body);
@@ -384,10 +398,8 @@ const string Master::Http::REDIRECT_HELP = HELP(
"this will attempt to redirect to the private IP address."));
-Future<Response> Master::Http::redirect(const Request& request)
+Future<Response> Master::Http::redirect(const Request& request) const
{
- LOG(INFO) << "HTTP request for '" << request.path << "'";
-
// If there's no leader, redirect to this master's base url.
MasterInfo info = master->leader.isSome()
? master->leader.get()
@@ -418,9 +430,8 @@ const string Master::Http::SLAVES_HELP = HELP(
"this master formated as a json object."));
-Future<Response> Master::Http::slaves(const Request& request) {
- LOG(INFO) << "HTTP request for '" << request.path << "'";
-
+Future<Response> Master::Http::slaves(const Request& request) const
+{
JSON::Object object;
{
@@ -439,10 +450,8 @@ Future<Response> Master::Http::slaves(const Request& request) {
}
-Future<Response> Master::Http::state(const Request& request)
+Future<Response> Master::Http::state(const Request& request) const
{
- LOG(INFO) << "HTTP request for '" << request.path << "'";
-
JSON::Object object;
object.values["version"] = MESOS_VERSION;
@@ -725,10 +734,8 @@ private:
};
-Future<Response> Master::Http::stateSummary(const Request& request)
+Future<Response> Master::Http::stateSummary(const Request& request) const
{
- LOG(INFO) << "HTTP request for '" << request.path << "'";
-
JSON::Object object;
object.values["hostname"] = master->info().hostname();
@@ -835,10 +842,8 @@ Future<Response> Master::Http::stateSummary(const Request& request)
}
-Future<Response> Master::Http::roles(const Request& request)
+Future<Response> Master::Http::roles(const Request& request) const
{
- LOG(INFO) << "HTTP request for '" << request.path << "'";
-
JSON::Object object;
// Model all of the roles.
@@ -866,7 +871,7 @@ const string Master::Http::SHUTDOWN_HELP = HELP(
"Returns 200 OK if the framework was correctly shutdown."));
-Future<Response> Master::Http::shutdown(const Request& request)
+Future<Response> Master::Http::shutdown(const Request& request) const
{
if (request.method != "POST") {
return BadRequest("Expecting POST");
@@ -932,7 +937,7 @@ Future<Response> Master::Http::shutdown(const Request& request)
Future<Response> Master::Http::_shutdown(
const FrameworkID& id,
- bool authorized)
+ bool authorized) const
{
if (!authorized) {
return Unauthorized("Mesos master");
@@ -1013,10 +1018,8 @@ struct TaskComparator
};
-Future<Response> Master::Http::tasks(const Request& request)
+Future<Response> Master::Http::tasks(const Request& request) const
{
- LOG(INFO) << "HTTP request for '" << request.path << "'";
-
// Get list options (limit and offset).
Result<int> result = numify<int>(request.query.get("limit"));
size_t limit = result.isSome() ? result.get() : TASK_LIMIT;
@@ -1075,7 +1078,7 @@ Future<Response> Master::Http::tasks(const Request& request)
}
-Result<Credential> Master::Http::authenticate(const Request& request)
+Result<Credential> Master::Http::authenticate(const Request& request) const
{
// By default, assume everyone is authenticated if no credentials
// were provided.
http://git-wip-us.apache.org/repos/asf/mesos/blob/115db925/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 49638ce..4ad683e 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -38,6 +38,7 @@
#include <process/collect.hpp>
#include <process/defer.hpp>
#include <process/delay.hpp>
+#include <process/http.hpp>
#include <process/id.hpp>
#include <process/limiter.hpp>
#include <process/owned.hpp>
@@ -111,6 +112,8 @@ namespace mesos {
namespace internal {
namespace master {
+namespace http = process::http;
+
using mesos::master::RoleInfo;
using mesos::master::allocator::Allocator;
@@ -277,7 +280,6 @@ Master::Master(
const Option<shared_ptr<RateLimiter>>& _slaveRemovalLimiter,
const Flags& _flags)
: ProcessBase("master"),
- http(this),
flags(_flags),
allocator(_allocator),
registrar(_registrar),
@@ -721,33 +723,60 @@ void Master::initialize()
&AuthenticateMessage::pid);
// Setup HTTP routes.
+ Http http = Http(this);
+
route("/health",
Http::HEALTH_HELP,
- lambda::bind(&Http::health, http, lambda::_1));
+ [http](const http::Request& request) {
+ return http.health(request);
+ });
route("/observe",
Http::OBSERVE_HELP,
- lambda::bind(&Http::observe, http, lambda::_1));
+ [http](const http::Request& request) {
+ Http::log(request);
+ return http.observe(request);
+ });
route("/redirect",
Http::REDIRECT_HELP,
- lambda::bind(&Http::redirect, http, lambda::_1));
+ [http](const http::Request& request) {
+ return http.redirect(request);
+ });
route("/roles.json",
None(),
- lambda::bind(&Http::roles, http, lambda::_1));
+ [http](const http::Request& request) {
+ Http::log(request);
+ return http.roles(request);
+ });
route("/shutdown",
Http::SHUTDOWN_HELP,
- lambda::bind(&Http::shutdown, http, lambda::_1));
+ [http](const http::Request& request) {
+ Http::log(request);
+ return http.shutdown(request);
+ });
route("/slaves",
Http::SLAVES_HELP,
- lambda::bind(&Http::slaves, http, lambda::_1));
+ [http](const http::Request& request) {
+ Http::log(request);
+ return http.slaves(request);
+ });
route("/state.json",
None(),
- lambda::bind(&Http::state, http, lambda::_1));
+ [http](const http::Request& request) {
+ Http::log(request);
+ return http.state(request);
+ });
route("/state-summary",
None(),
- lambda::bind(&Http::stateSummary, http, lambda::_1));
+ [http](const http::Request& request) {
+ Http::log(request);
+ return http.stateSummary(request);
+ });
route("/tasks.json",
Http::TASKS_HELP,
- lambda::bind(&Http::tasks, http, lambda::_1));
+ [http](const http::Request& request) {
+ Http::log(request);
+ return http.tasks(request);
+ });
// Provide HTTP assets from a "webui" directory. This is either
// specified via flags (which is necessary for running out of the
http://git-wip-us.apache.org/repos/asf/mesos/blob/115db925/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index d00be17..ee53dfb 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -477,41 +477,45 @@ private:
public:
explicit Http(Master* _master) : master(_master) {}
+ // Logs the request, route handlers can compose this with the
+ // desired request handler to get consistent request logging.
+ static void log(const process::http::Request& request);
+
// /master/health
process::Future<process::http::Response> health(
- const process::http::Request& request);
+ const process::http::Request& request) const;
// /master/observe
process::Future<process::http::Response> observe(
- const process::http::Request& request);
+ const process::http::Request& request) const;
// /master/redirect
process::Future<process::http::Response> redirect(
- const process::http::Request& request);
+ const process::http::Request& request) const;
// /master/roles.json
process::Future<process::http::Response> roles(
- const process::http::Request& request);
+ const process::http::Request& request) const;
// /master/shutdown
process::Future<process::http::Response> shutdown(
- const process::http::Request& request);
+ const process::http::Request& request) const;
// /master/slaves
process::Future<process::http::Response> slaves(
- const process::http::Request& request);
+ const process::http::Request& request) const;
// /master/state.json
process::Future<process::http::Response> state(
- const process::http::Request& request);
+ const process::http::Request& request) const;
// /master/state-summary
process::Future<process::http::Response> stateSummary(
- const process::http::Request& request);
+ const process::http::Request& request) const;
// /master/tasks.json
process::Future<process::http::Response> tasks(
- const process::http::Request& request);
+ const process::http::Request& request) const;
const static std::string HEALTH_HELP;
const static std::string OBSERVE_HELP;
@@ -524,15 +528,16 @@ private:
// Helper for doing authentication, returns the credential used if
// the authentication was successful (or none if no credentials
// have been given to the master), otherwise an Error.
- Result<Credential> authenticate(const process::http::Request& request);
+ Result<Credential> authenticate(
+ const process::http::Request& request) const;
// Continuations.
process::Future<process::http::Response> _shutdown(
const FrameworkID& id,
- bool authorized = true);
+ bool authorized = true) const;
Master* master;
- } http;
+ };
Master(const Master&); // No copying.
Master& operator = (const Master&); // No assigning.
http://git-wip-us.apache.org/repos/asf/mesos/blob/115db925/src/slave/http.cpp
----------------------------------------------------------------------
diff --git a/src/slave/http.cpp b/src/slave/http.cpp
index f678aab..b5e77b0 100644
--- a/src/slave/http.cpp
+++ b/src/slave/http.cpp
@@ -229,6 +229,22 @@ JSON::Object model(const Framework& framework)
}
+void Slave::Http::log(const Request& request)
+{
+ Option<string> userAgent = request.headers.get("User-Agent");
+ Option<string> forwardedFor = request.headers.get("X-Forwarded-For");
+
+ LOG(INFO) << "HTTP " << request.method << " for " << request.path
+ << " from " << request.client
+ << (userAgent.isSome()
+ ? " with User-Agent='" + userAgent.get() + "'"
+ : "")
+ << (forwardedFor.isSome()
+ ? " with X-Forwarded-For='" + forwardedFor.get() + "'"
+ : "");
+}
+
+
const string Slave::Http::HEALTH_HELP = HELP(
TLDR(
"Health check of the Slave."),
@@ -239,16 +255,14 @@ const string Slave::Http::HEALTH_HELP = HELP(
"Delayed responses are also indicative of poor health."));
-Future<Response> Slave::Http::health(const Request& request)
+Future<Response> Slave::Http::health(const Request& request) const
{
return OK();
}
-Future<Response> Slave::Http::state(const Request& request)
+Future<Response> Slave::Http::state(const Request& request) const
{
- LOG(INFO) << "HTTP request for '" << request.path << "'";
-
JSON::Object object;
object.values["version"] = MESOS_VERSION;
http://git-wip-us.apache.org/repos/asf/mesos/blob/115db925/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index c78ee3c..bf290bf 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -39,6 +39,7 @@
#include <process/defer.hpp>
#include <process/delay.hpp>
#include <process/dispatch.hpp>
+#include <process/http.hpp>
#include <process/id.hpp>
#include <process/time.hpp>
@@ -105,6 +106,8 @@ namespace mesos {
namespace internal {
namespace slave {
+namespace http = process::http;
+
using namespace state;
Slave::Slave(const slave::Flags& _flags,
@@ -115,7 +118,6 @@ Slave::Slave(const slave::Flags& _flags,
StatusUpdateManager* _statusUpdateManager)
: ProcessBase(process::ID::generate("slave")),
state(RECOVERING),
- http(this),
flags(_flags),
completedFrameworks(MAX_COMPLETED_FRAMEWORKS),
detector(_detector),
@@ -462,10 +464,19 @@ void Slave::initialize()
&PingSlaveMessage::connected);
// Setup HTTP routes.
+ Http http = Http(this);
+
route("/health",
Http::HEALTH_HELP,
- lambda::bind(&Http::health, http, lambda::_1));
- route("/state.json", None(), lambda::bind(&Http::state, http, lambda::_1));
+ [http](const http::Request& request) {
+ return http.health(request);
+ });
+ route("/state.json",
+ None(),
+ [http](const http::Request& request) {
+ Http::log(request);
+ return http.state(request);
+ });
// Expose the log file for the webui. Fall back to 'log_dir' if
// an explicit file was not specified.
http://git-wip-us.apache.org/repos/asf/mesos/blob/115db925/src/slave/slave.hpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.hpp b/src/slave/slave.hpp
index 654a869..adb52b5 100644
--- a/src/slave/slave.hpp
+++ b/src/slave/slave.hpp
@@ -373,19 +373,23 @@ private:
public:
explicit Http(Slave* _slave) : slave(_slave) {}
+ // Logs the request, route handlers can compose this with the
+ // desired request handler to get consistent request logging.
+ static void log(const process::http::Request& request);
+
// /slave/health
process::Future<process::http::Response> health(
- const process::http::Request& request);
+ const process::http::Request& request) const;
// /slave/state.json
process::Future<process::http::Response> state(
- const process::http::Request& request);
+ const process::http::Request& request) const;
static const std::string HEALTH_HELP;
private:
Slave* slave;
- } http;
+ };
friend struct Framework;
friend struct Executor;
[2/3] mesos git commit: Added the client address to http::Request.
Posted by bm...@apache.org.
Added the client address to http::Request.
Review: https://reviews.apache.org/r/33934
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/21ee7fa5
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/21ee7fa5
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/21ee7fa5
Branch: refs/heads/master
Commit: 21ee7fa5fba8c5e7e47b4dd34e15056720b1ee3b
Parents: 389995d
Author: Benjamin Mahler <be...@gmail.com>
Authored: Wed May 6 22:13:05 2015 -0700
Committer: Benjamin Mahler <be...@gmail.com>
Committed: Thu May 7 20:04:01 2015 -0700
----------------------------------------------------------------------
3rdparty/libprocess/include/process/http.hpp | 5 +++++
3rdparty/libprocess/src/process.cpp | 8 +++++++-
3rdparty/libprocess/src/tests/http_tests.cpp | 3 +++
3 files changed, 15 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/21ee7fa5/3rdparty/libprocess/include/process/http.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/include/process/http.hpp b/3rdparty/libprocess/include/process/http.hpp
index 5210ce0..058fa02 100644
--- a/3rdparty/libprocess/include/process/http.hpp
+++ b/3rdparty/libprocess/include/process/http.hpp
@@ -10,6 +10,7 @@
#include <string>
#include <vector>
+#include <process/address.hpp>
#include <process/future.hpp>
#include <process/owned.hpp>
#include <process/pid.hpp>
@@ -45,6 +46,10 @@ void initialize();
struct Request
{
+ // Contains the client's address. Note that this may
+ // correspond to a proxy or load balancer address.
+ network::Address client;
+
// TODO(benh): Add major/minor version.
// TODO(bmahler): Header names are not case sensitive! Either make these
// case-insensitive, or add a variable for each header in HTTP 1.0/1.1 (like
http://git-wip-us.apache.org/repos/asf/mesos/blob/21ee7fa5/3rdparty/libprocess/src/process.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/process.cpp b/3rdparty/libprocess/src/process.cpp
index 8ab0dbc..588bd3e 100644
--- a/3rdparty/libprocess/src/process.cpp
+++ b/3rdparty/libprocess/src/process.cpp
@@ -567,11 +567,17 @@ void decode_recv(
delete socket;
return;
}
+
// Decode as much of the data as possible into HTTP requests.
- const deque<Request*>& requests = decoder->decode(data, length.get());
+ deque<Request*> requests = decoder->decode(data, length.get());
if (!requests.empty()) {
foreach (Request* request, requests) {
+ // Augment each Request with the client's address. This should
+ // never fail since there remains a reference to this Socket!
+ Try<Address> address = socket->address();
+ CHECK_SOME(address);
+ request->client = address.get();
process_manager->handle(decoder->socket(), request);
}
} else if (requests.empty() && decoder->failed()) {
http://git-wip-us.apache.org/repos/asf/mesos/blob/21ee7fa5/3rdparty/libprocess/src/tests/http_tests.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/tests/http_tests.cpp b/3rdparty/libprocess/src/tests/http_tests.cpp
index dfdb233..d29cd29 100644
--- a/3rdparty/libprocess/src/tests/http_tests.cpp
+++ b/3rdparty/libprocess/src/tests/http_tests.cpp
@@ -7,6 +7,7 @@
#include <string>
+#include <process/address.hpp>
#include <process/future.hpp>
#include <process/gmock.hpp>
#include <process/gtest.hpp>
@@ -368,6 +369,7 @@ TEST(HTTP, PathParse)
http::Response validateGetWithoutQuery(const http::Request& request)
{
+ EXPECT_NE(network::Address(), request.client);
EXPECT_EQ("GET", request.method);
EXPECT_THAT(request.path, EndsWith("get"));
EXPECT_EQ("", request.body);
@@ -380,6 +382,7 @@ http::Response validateGetWithoutQuery(const http::Request& request)
http::Response validateGetWithQuery(const http::Request& request)
{
+ EXPECT_NE(network::Address(), request.client);
EXPECT_EQ("GET", request.method);
EXPECT_THAT(request.path, EndsWith("get"));
EXPECT_EQ("", request.body);